From 4294face784e9b2eff46702371baa7ae5ec9fdfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Thu, 10 Apr 2025 12:29:54 +0200 Subject: [PATCH] feat(scrapeURL): reintroduce default timeout for simple queries (#1440) * feat(scrapeURL): reintroduce default timeout * fix * adjust timeouts --- apps/api/src/__tests__/snips/batch-scrape.test.ts | 4 ++-- apps/api/src/scraper/scrapeURL/index.ts | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/api/src/__tests__/snips/batch-scrape.test.ts b/apps/api/src/__tests__/snips/batch-scrape.test.ts index 1890b08b..f3e9e585 100644 --- a/apps/api/src/__tests__/snips/batch-scrape.test.ts +++ b/apps/api/src/__tests__/snips/batch-scrape.test.ts @@ -8,7 +8,7 @@ describe("Batch scrape tests", () => { expect(response.body.data[0]).toHaveProperty("markdown"); expect(response.body.data[0].markdown).toContain("Firecrawl"); - }, 30000); + }, 180000); if (!process.env.TEST_SUITE_SELF_HOSTED) { describe("JSON format", () => { @@ -45,7 +45,7 @@ describe("Batch scrape tests", () => { expect(response.body.data[0].json).toHaveProperty("is_open_source"); expect(response.body.data[0].json.is_open_source).toBe(true); expect(typeof response.body.data[0].json.is_open_source).toBe("boolean"); - }, 30000); + }, 180000); }); } }); diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts index e047d1cb..cedd275e 100644 --- a/apps/api/src/scraper/scrapeURL/index.ts +++ b/apps/api/src/scraper/scrapeURL/index.ts @@ -230,7 +230,9 @@ async function scrapeURLLoop(meta: Meta): Promise { const timeToRun = meta.options.timeout !== undefined ? Math.round(meta.options.timeout / Math.min(fallbackList.length, 2)) - : undefined; + : (!meta.options.actions && !meta.options.jsonOptions && !meta.options.extract) + ? Math.round(120000 / Math.min(fallbackList.length, 2)) + : undefined; for (const { engine, unsupportedFeatures } of fallbackList) { meta.internalOptions.abort?.throwIfAborted();