Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(WebScraper): always report error of last scraper in order

Browse Source
This commit is contained in:
Gergő Móricz
2024-09-24 20:03:49 +02:00
parent 3e661a2087
commit 43730b5db6
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/single_url.ts
+5 -1
View File
@@ -392,8 +392,12 @@ export async function scrapSingleUrl(
if (attempt.pageStatusCode) { if (attempt.pageStatusCode) {
pageStatusCode = attempt.pageStatusCode; pageStatusCode = attempt.pageStatusCode;
} }
if (attempt.pageError && attempt.pageStatusCode >= 400) { if (attempt.pageError && (attempt.pageStatusCode >= 400 || scrapersInOrder.indexOf(scraper) === scrapersInOrder.length - 1)) { // force pageError if it's the last scraper and it failed too
pageError = attempt.pageError; pageError = attempt.pageError;
if (attempt.pageStatusCode < 400 || !attempt.pageStatusCode) {
pageStatusCode = 500;
}
} else if (attempt && attempt.pageStatusCode && attempt.pageStatusCode < 400) { } else if (attempt && attempt.pageStatusCode && attempt.pageStatusCode < 400) {
pageError = undefined; pageError = undefined;
} }