Nick:

apps/api/src/controllers/v0/scrape.ts

@@ -255,12 +255,25 @@ export async function scrapeController(req: Request, res: Response) {
       }
     }
     
+    let doc = result.data;
+    if (!pageOptions || !pageOptions.includeRawHtml) {
+      if (doc && doc.rawHtml) {
+        delete doc.rawHtml;
+      }
+    }
+  
+    if(pageOptions && pageOptions.includeExtract) {
+      if(!pageOptions.includeMarkdown && doc && doc.markdown) {
+        delete doc.markdown;
+      }
+    }
+
     logJob({
       job_id: jobId,
       success: result.success,
       message: result.error,
       num_docs: 1,
-      docs: [result.data],
+      docs: [doc],
       time_taken: timeTakenInSeconds,
       team_id: team_id,
       mode: "scrape",

apps/api/src/controllers/v1/scrape.ts

@@ -115,6 +115,18 @@ export async function scrapeController(
     });
   }
 
+  if (!pageOptions || !pageOptions.includeRawHtml) {
+    if (doc && doc.rawHtml) {
+      delete doc.rawHtml;
+    }
+  }
+
+  if(pageOptions && pageOptions.includeExtract) {
+    if(!pageOptions.includeMarkdown && doc && doc.markdown) {
+      delete doc.markdown;
+    }
+  }
+
   logJob({
     job_id: jobId,
     success: true,

apps/api/src/services/queue-worker.ts

@@ -202,24 +202,6 @@ async function processJob(job: Job, token: string) {
 
     const rawHtml = docs[0] ? docs[0].rawHtml : "";
 
-    if (job.data.crawl_id && (!job.data.pageOptions || !job.data.pageOptions.includeRawHtml)) {
-      if (docs[0] && docs[0].rawHtml) {
-        delete docs[0].rawHtml;
-      }
-    }
-
-    if(job.data.pageOptions && job.data.pageOptions.includeExtract ) {
-      if(!job.data.pageOptions.includeMarkdown) {
-        delete docs[0].markdown;
-      }
-      // if(!job.data.pageOptions.includeRawHtml) {
-      //   delete docs[0].rawHtml;
-      // }
-      // if(!job.data.pageOptions.includeHtml) {
-      //   delete docs[0].html;
-      // }
-    }
-
     const data = {
       success,
       result: {