From 5818236659a9c3ceef45488e7f439897cb2d8383 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Tue, 20 Aug 2024 22:51:12 +0200
Subject: [PATCH] fix: remove rawHtml properly

---
 apps/api/src/main/runWebScraper.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index 84826bdd..b28d039e 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -45,6 +45,9 @@ export async function startWebScraperPipeline({
     },
     onSuccess: (result, mode) => {
       Logger.debug(`🐂 Job completed ${job.id}`);
+      if (job.data.crawl_id && (!job.data.pageOptions || !job.data.pageOptions.includeRawHtml)) {
+        delete result[0].rawHtml;
+      }
       saveJob(job, result, token, mode);
     },
     onError: (error) => {