Nick:

apps/api/src/services/logging/scrape_log.ts

@@ -1,32 +1,42 @@
 import "dotenv/config";
 import { ScrapeLog } from "../../types";
 import { supabase_service } from "../supabase";
+import { PageOptions } from "../../lib/entities";
 
-export async function logScrape(scrapeLog: ScrapeLog) {
+export async function logScrape(
+  scrapeLog: ScrapeLog,
+  pageOptions?: PageOptions
+) {
   try {
     // Only log jobs in production
     // if (process.env.ENV !== "production") {
     //   return;
     // }
+    // Redact any pages that have an authorization header
+    if (
+      pageOptions &&
+      pageOptions.headers &&
+      pageOptions.headers["Authorization"]
+    ) {
+      scrapeLog.html = "REDACTED DUE TO AUTHORIZATION HEADER";
+    }
 
-    const { data, error } = await supabase_service
-      .from("scrape_logs")
-      .insert([
-        {
-          url: scrapeLog.url,
-          scraper: scrapeLog.scraper,
-          success: scrapeLog.success,
-          response_code: scrapeLog.response_code,
-          time_taken_seconds: scrapeLog.time_taken_seconds,
-          proxy: scrapeLog.proxy,
-          retried: scrapeLog.retried,
-          error_message: scrapeLog.error_message,
-          date_added: new Date().toISOString(),
-          html: scrapeLog.html,
-          ipv4_support: scrapeLog.ipv4_support,
-          ipv6_support: scrapeLog.ipv6_support,
-        },
-      ]);
+    const { data, error } = await supabase_service.from("scrape_logs").insert([
+      {
+        url: scrapeLog.url,
+        scraper: scrapeLog.scraper,
+        success: scrapeLog.success,
+        response_code: scrapeLog.response_code,
+        time_taken_seconds: scrapeLog.time_taken_seconds,
+        proxy: scrapeLog.proxy,
+        retried: scrapeLog.retried,
+        error_message: scrapeLog.error_message,
+        date_added: new Date().toISOString(),
+        html: scrapeLog.html,
+        ipv4_support: scrapeLog.ipv4_support,
+        ipv6_support: scrapeLog.ipv6_support,
+      },
+    ]);
 
     if (error) {
       console.error("Error logging proxy:\n", error);