Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update fetch.ts

Browse Source
This commit is contained in:
Nicolas
2024-10-03 18:55:39 -03:00
parent 25dd16bf2a
commit 668ff3c71b
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/scrapers/fetch.ts
+2 -2
View File
@@ -51,7 +51,7 @@ export async function scrapWithFetch(
const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF); const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF);
logParams.response_code = pageStatusCode; logParams.response_code = pageStatusCode;
logParams.error_message = pageError; logParams.error_message = pageError;
return { content, pageStatusCode, pageError }; return { content, pageStatusCode: response.status, pageError };
} else { } else {
const text = response.data; const text = response.data;
logParams.success = true; logParams.success = true;
@@ -67,7 +67,7 @@ export async function scrapWithFetch(
logParams.error_message = error.message || error; logParams.error_message = error.message || error;
Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} | Error: ${error}`); Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} | Error: ${error}`);
} }
return { content: "", pageStatusCode: null, pageError: logParams.error_message }; return { content: "", pageStatusCode: error.response?.status ?? null, pageError: logParams.error_message };
} finally { } finally {
const endTime = Date.now(); const endTime = Date.now();
logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000; logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;