diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts index 84826bdd..b28d039e 100644 --- a/apps/api/src/main/runWebScraper.ts +++ b/apps/api/src/main/runWebScraper.ts @@ -45,6 +45,9 @@ export async function startWebScraperPipeline({ }, onSuccess: (result, mode) => { Logger.debug(`🐂 Job completed ${job.id}`); + if (job.data.crawl_id && (!job.data.pageOptions || !job.data.pageOptions.includeRawHtml)) { + delete result[0].rawHtml; + } saveJob(job, result, token, mode); }, onError: (error) => {