From bdbc05a4c7c3723e688fb63a191c1ebcff070618 Mon Sep 17 00:00:00 2001 From: rafaelmmiller <150964962+rafaelsideguide@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:33:39 -0300 Subject: [PATCH 1/4] added check for object and trycatch as workaround for 502s --- apps/js-sdk/firecrawl/package.json | 2 +- apps/js-sdk/firecrawl/src/index.ts | 72 ++++++++++++++++-------------- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 30277cc3..74dfcb02 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "1.9.3", + "version": "1.9.4", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 37fc5ef0..a54506ad 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -462,7 +462,7 @@ export default class FirecrawlApp { let statusData = response.data if ("data" in statusData) { let data = statusData.data; - while ('next' in statusData) { + while (typeof statusData === 'object' && 'next' in statusData) { statusData = (await this.getRequest(statusData.next, headers)).data; data = data.concat(statusData.data); } @@ -691,7 +691,7 @@ export default class FirecrawlApp { let statusData = response.data if ("data" in statusData) { let data = statusData.data; - while ('next' in statusData) { + while (typeof statusData === 'object' && 'next' in statusData) { statusData = (await this.getRequest(statusData.next, headers)).data; data = data.concat(statusData.data); } @@ -850,42 +850,46 @@ export default class FirecrawlApp { headers: AxiosRequestHeaders, checkInterval: number ): Promise { - while (true) { - let statusResponse: AxiosResponse = await this.getRequest( - `${this.apiUrl}/v1/crawl/${id}`, - headers - ); - if (statusResponse.status === 200) { - let statusData = statusResponse.data; - if (statusData.status === "completed") { - if ("data" in statusData) { - let data = statusData.data; - while ('next' in statusData) { - statusResponse = await this.getRequest(statusData.next, headers); - statusData = statusResponse.data; - data = data.concat(statusData.data); + try { + while (true) { + let statusResponse: AxiosResponse = await this.getRequest( + `${this.apiUrl}/v1/crawl/${id}`, + headers + ); + if (statusResponse.status === 200) { + let statusData = statusResponse.data; + if (statusData.status === "completed") { + if ("data" in statusData) { + let data = statusData.data; + while (typeof statusData === 'object' && 'next' in statusData) { + statusResponse = await this.getRequest(statusData.next, headers); + statusData = statusResponse.data; + data = data.concat(statusData.data); + } + statusData.data = data; + return statusData; + } else { + throw new FirecrawlError("Crawl job completed but no data was returned", 500); } - statusData.data = data; - return statusData; - } else { - throw new FirecrawlError("Crawl job completed but no data was returned", 500); - } - } else if ( - ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status) - ) { - checkInterval = Math.max(checkInterval, 2); - await new Promise((resolve) => - setTimeout(resolve, checkInterval * 1000) - ); + } else if ( + ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status) + ) { + checkInterval = Math.max(checkInterval, 2); + await new Promise((resolve) => + setTimeout(resolve, checkInterval * 1000) + ); + } else { + throw new FirecrawlError( + `Crawl job failed or was stopped. Status: ${statusData.status}`, + 500 + ); + } } else { - throw new FirecrawlError( - `Crawl job failed or was stopped. Status: ${statusData.status}`, - 500 - ); + this.handleError(statusResponse, "check crawl status"); } - } else { - this.handleError(statusResponse, "check crawl status"); } + } catch (error: any) { + throw new FirecrawlError(error, 500); } } From 4f65d350a3f98211c18d1695a3473e5d27e6ee6f Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 26 Dec 2024 12:52:52 -0300 Subject: [PATCH 2/4] Update package.json --- apps/js-sdk/firecrawl/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 74dfcb02..29679b8b 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "1.9.4", + "version": "1.10.1", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", From f15ef0e7582e39e6ed17ee9f01e10969f02b4c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Thu, 26 Dec 2024 20:29:09 +0100 Subject: [PATCH 3/4] feat(scrapeURL/fire-engine/chrome-cdp): handle file downloads --- .../engines/fire-engine/checkStatus.ts | 13 ++++++- .../scrapeURL/engines/fire-engine/index.ts | 39 ++++++++----------- apps/api/src/scraper/scrapeURL/error.ts | 8 ++++ apps/api/src/scraper/scrapeURL/index.ts | 5 +++ 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts index 6f65db98..e02e9dbb 100644 --- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts +++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts @@ -3,7 +3,7 @@ import * as Sentry from "@sentry/node"; import { z } from "zod"; import { robustFetch } from "../../lib/fetch"; -import { ActionError, EngineError, SiteError } from "../../error"; +import { ActionError, EngineError, SiteError, UnsupportedFileError } from "../../error"; const successSchema = z.object({ jobId: z.string(), @@ -35,6 +35,12 @@ const successSchema = z.object({ }) .array() .optional(), + + // chrome-cdp only -- file download handler + file: z.object({ + name: z.string(), + content: z.string(), + }).optional().or(z.null()), }); export type FireEngineCheckStatusSuccess = z.infer; @@ -111,6 +117,11 @@ export async function fireEngineCheckStatus( status.error.includes("Chrome error: ") ) { throw new SiteError(status.error.split("Chrome error: ")[1]); + } else if ( + typeof status.error === "string" && + status.error.includes("File size exceeds") + ) { + throw new UnsupportedFileError("File size exceeds " + status.error.split("File size exceeds ")[1]); } else if ( typeof status.error === "string" && // TODO: improve this later diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts index d753465d..aa869836 100644 --- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts @@ -13,7 +13,7 @@ import { FireEngineCheckStatusSuccess, StillProcessingError, } from "./checkStatus"; -import { ActionError, EngineError, SiteError, TimeoutError } from "../../error"; +import { ActionError, EngineError, SiteError, TimeoutError, UnsupportedFileError } from "../../error"; import * as Sentry from "@sentry/node"; import { Action } from "../../../../lib/entities"; import { specialtyScrapeCheck } from "../utils/specialtyHandler"; @@ -71,7 +71,8 @@ async function performFireEngineScrape< } else if ( error instanceof EngineError || error instanceof SiteError || - error instanceof ActionError + error instanceof ActionError || + error instanceof UnsupportedFileError ) { logger.debug("Fire-engine scrape job failed.", { error, @@ -91,6 +92,19 @@ async function performFireEngineScrape< await new Promise((resolve) => setTimeout(resolve, 250)); } + specialtyScrapeCheck( + logger.child({ + method: "performFireEngineScrape/specialtyScrapeCheck", + }), + status.responseHeaders, + ); + + if (status.file) { + const content = status.file.content; + delete status.file; + status.content = Buffer.from(content, "base64").toString("utf8"); // TODO: handle other encodings via Content-Type tag + } + return status; } @@ -160,13 +174,6 @@ export async function scrapeURLWithFireEngineChromeCDP( timeout, ); - specialtyScrapeCheck( - meta.logger.child({ - method: "scrapeURLWithFireEngineChromeCDP/specialtyScrapeCheck", - }), - response.responseHeaders, - ); - if ( meta.options.formats.includes("screenshot") || meta.options.formats.includes("screenshot@fullPage") @@ -241,13 +248,6 @@ export async function scrapeURLWithFireEnginePlaywright( timeout, ); - specialtyScrapeCheck( - meta.logger.child({ - method: "scrapeURLWithFireEnginePlaywright/specialtyScrapeCheck", - }), - response.responseHeaders, - ); - if (!response.url) { meta.logger.warn("Fire-engine did not return the response's URL", { response, @@ -301,13 +301,6 @@ export async function scrapeURLWithFireEngineTLSClient( timeout, ); - specialtyScrapeCheck( - meta.logger.child({ - method: "scrapeURLWithFireEngineTLSClient/specialtyScrapeCheck", - }), - response.responseHeaders, - ); - if (!response.url) { meta.logger.warn("Fire-engine did not return the response's URL", { response, diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts index 689f90c8..bff3a492 100644 --- a/apps/api/src/scraper/scrapeURL/error.ts +++ b/apps/api/src/scraper/scrapeURL/error.ts @@ -64,3 +64,11 @@ export class ActionError extends Error { this.code = code; } } + +export class UnsupportedFileError extends Error { + public reason: string; + constructor(reason: string) { + super("Scrape resulted in unsupported file: " + reason); + this.reason = reason; + } +} diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts index 1df812bd..130ef9ee 100644 --- a/apps/api/src/scraper/scrapeURL/index.ts +++ b/apps/api/src/scraper/scrapeURL/index.ts @@ -19,6 +19,7 @@ import { RemoveFeatureError, SiteError, TimeoutError, + UnsupportedFileError, } from "./error"; import { executeTransformers } from "./transformers"; import { LLMRefusalError } from "./transformers/llmExtract"; @@ -292,6 +293,8 @@ async function scrapeURLLoop(meta: Meta): Promise { throw error; } else if (error instanceof ActionError) { throw error; + } else if (error instanceof UnsupportedFileError) { + throw error; } else { Sentry.captureException(error); meta.logger.info( @@ -414,6 +417,8 @@ export async function scrapeURL( meta.logger.warn("scrapeURL: Site failed to load in browser", { error }); } else if (error instanceof ActionError) { meta.logger.warn("scrapeURL: Action(s) failed to complete", { error }); + } else if (error instanceof UnsupportedFileError) { + meta.logger.warn("scrapeURL: Tried to scrape unsupported file", { error }); } else { Sentry.captureException(error); meta.logger.error("scrapeURL: Unexpected error happened", { error }); From c543f4f76c0fd187be684b8571d1528dad84744e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Thu, 26 Dec 2024 20:31:51 +0100 Subject: [PATCH 4/4] feat(scrapeURL/pdf): update mock Blob implementation to pass TypeScript --- apps/api/src/scraper/scrapeURL/engines/pdf/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts index 9d2f11b1..6bac2ba4 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts @@ -32,6 +32,9 @@ async function scrapePDFWithLlamaParse( tempFilePath, ) as unknown as ReadableStream; }, + bytes() { + throw Error("Unimplemented in mock Blob: bytes"); + }, arrayBuffer() { throw Error("Unimplemented in mock Blob: arrayBuffer"); },