Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scrapeURL): reintroduce default timeout for simple queries (#1440)

Browse Source 
* feat(scrapeURL): reintroduce default timeout

* fix

* adjust timeouts
This commit is contained in:
Gergő Móricz
2025-04-10 12:29:54 +02:00
committed by GitHub
parent 7bb5643028
commit 4294face78
2 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/__tests__/snips/batch-scrape.test.ts
+2 -2
View File
@@ -8,7 +8,7 @@ describe("Batch scrape tests", () => {
expect(response.body.data[0]).toHaveProperty("markdown"); expect(response.body.data[0]).toHaveProperty("markdown");
expect(response.body.data[0].markdown).toContain("Firecrawl"); expect(response.body.data[0].markdown).toContain("Firecrawl");
}, 30000); }, 180000);
if (!process.env.TEST_SUITE_SELF_HOSTED) { if (!process.env.TEST_SUITE_SELF_HOSTED) {
describe("JSON format", () => { describe("JSON format", () => {
@@ -45,7 +45,7 @@ describe("Batch scrape tests", () => {
expect(response.body.data[0].json).toHaveProperty("is_open_source"); expect(response.body.data[0].json).toHaveProperty("is_open_source");
expect(response.body.data[0].json.is_open_source).toBe(true); expect(response.body.data[0].json.is_open_source).toBe(true);
expect(typeof response.body.data[0].json.is_open_source).toBe("boolean"); expect(typeof response.body.data[0].json.is_open_source).toBe("boolean");
}, 30000); }, 180000);
}); });
} }
}); });
apps/api/src/scraper/scrapeURL/index.ts
+2
View File
@@ -230,6 +230,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
const timeToRun = const timeToRun =
meta.options.timeout !== undefined meta.options.timeout !== undefined
? Math.round(meta.options.timeout / Math.min(fallbackList.length, 2)) ? Math.round(meta.options.timeout / Math.min(fallbackList.length, 2))
: (!meta.options.actions && !meta.options.jsonOptions && !meta.options.extract)
? Math.round(120000 / Math.min(fallbackList.length, 2))
: undefined; : undefined;
for (const { engine, unsupportedFeatures } of fallbackList) { for (const { engine, unsupportedFeatures } of fallbackList) {