Merge branch 'main' into v1/llm-extract

apps/api/src/controllers/v0/crawl-status.ts

@@ -56,6 +56,19 @@ export async function crawlStatusController(req: Request, res: Response) {
 
     const data = jobs.map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue);
 
+    if (
+      jobs.length > 0 &&
+      jobs[0].data &&
+      jobs[0].data.pageOptions &&
+      !jobs[0].data.pageOptions.includeRawHtml
+    ) {
+      data.forEach(item => {
+        if (item) {
+          delete item.rawHtml;
+        }
+      });
+    }
+
     res.json({
       status: jobStatus,
       current: jobStatuses.filter(x => x === "completed" || x === "failed").length,

apps/api/src/controllers/v1/crawl-status.ts

@@ -100,6 +100,14 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
     nextURL.searchParams.set("limit", req.query.limit);
   }
 
+  if (data.length > 0) {
+    if (!doneJobs[0].data.pageOptions.includeRawHtml) {
+      for (let ii = 0; ii < doneJobs.length; ii++) {
+        delete data[ii].rawHtml;
+      }
+    }
+  }
+
   res.status(200).json({
     status,
     completed: doneJobsLength,