WebScraper refactor into scrapeURL (#714)

* feat: use strictNullChecking * feat: switch logger to Winston * feat(scrapeURL): first batch * fix(scrapeURL): error swallow * fix(scrapeURL): add timeout to EngineResultsTracker * fix(scrapeURL): report unexpected error to sentry * chore: remove unused modules * feat(transfomers/coerce): warn when a format's response is missing * feat(scrapeURL): feature flag priorities, engine quality sorting, PDF and DOCX support * (add note) * feat(scrapeURL): wip readme * feat(scrapeURL): LLM extract * feat(scrapeURL): better warnings * fix(scrapeURL/engines/fire-engine;playwright): fix screenshot * feat(scrapeURL): add forceEngine internal option * feat(scrapeURL/engines): scrapingbee * feat(scrapeURL/transformars): uploadScreenshot * feat(scrapeURL): more intense tests * bunch of stuff * get rid of WebScraper (mostly) * adapt batch scrape * add staging deploy workflow * fix yaml * fix logger issues * fix v1 test schema * feat(scrapeURL/fire-engine/chrome-cdp): remove wait inserts on actions * scrapeURL: v0 backwards compat * logger fixes * feat(scrapeurl): v0 returnOnlyUrls support * fix(scrapeURL/v0): URL leniency * fix(batch-scrape): ts non-nullable * fix(scrapeURL/fire-engine/chromecdp): fix wait action * fix(logger): remove error debug key * feat(requests.http): use dotenv expression * fix(scrapeURL/extractMetadata): extract custom metadata * fix crawl option conversion * feat(scrapeURL): Add retry logic to robustFetch * fix(scrapeURL): crawl stuff * fix(scrapeURL): LLM extract * fix(scrapeURL/v0): search fix * fix(tests/v0): grant larger response size to v0 crawl status * feat(scrapeURL): basic fetch engine * feat(scrapeURL): playwright engine * feat(scrapeURL): add url-specific parameters * Update readme and examples * added e2e tests for most parameters. Still a few actions, location and iframes to be done. * fixed type * Nick: * Update scrape.ts * Update index.ts * added actions and base64 check * Nick: skipTls feature flag? * 403 * todo * todo * fixes * yeet headers from url specific params * add warning when final engine has feature deficit * expose engine results tracker for ScrapeEvents implementation * ingest scrape events * fixed some tests * comment * Update index.test.ts * fixed rawHtml * Update index.test.ts * update comments * move geolocation to global f-e option, fix removeBase64Images * Nick: * trim url-specific params * Update index.ts --------- Co-authored-by: Eric Ciarla <ericciarla@yahoo.com> Co-authored-by: rafaelmmiller <8574157+rafaelmmiller@users.noreply.github.com> Co-authored-by: Nicolas <nicolascamara29@gmail.com>
2024-11-07 20:57:33 +01:00
parent ed5a0d3cf2
commit 8d467c8ca7
142 changed files with 4230 additions and 6334 deletions
@@ -0,0 +1,66 @@
+import { ScrapingBeeClient } from "scrapingbee";
+import { Meta } from "../..";
+import { EngineScrapeResult } from "..";
+import { specialtyScrapeCheck } from "../utils/specialtyHandler";
+import { AxiosError, type AxiosResponse } from "axios";
+import { EngineError } from "../../error";
+
+const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY!);
+
+export function scrapeURLWithScrapingBee(wait_browser: "domcontentloaded" | "networkidle2"): ((meta: Meta) => Promise<EngineScrapeResult>) {
+    return async (meta: Meta): Promise<EngineScrapeResult> => {
+        let response: AxiosResponse<any>;
+        try {
+            response = await client.get({
+                url: meta.url,
+                params: {
+                    timeout: 15000, // TODO: dynamic timeout based on request timeout
+                    wait_browser: wait_browser,
+                    wait: Math.min(meta.options.waitFor, 35000),
+                    transparent_status_code: true,
+                    json_response: true,
+                    screenshot: meta.options.formats.includes("screenshot"),
+                    screenshot_full_page: meta.options.formats.includes("screenshot@fullPage"),             
+                },
+                headers: {
+                    "ScrapingService-Request": "TRUE", // this is sent to the page, not to ScrapingBee - mogery
+                },
+            });
+        } catch (error) {
+            if (error instanceof AxiosError && error.response !== undefined) {
+                response = error.response;
+            } else {
+                throw error;
+            }
+        }
+
+        const data: Buffer = response.data;
+        const body = JSON.parse(new TextDecoder().decode(data));
+
+        const headers = body.headers ?? {};
+        const isHiddenEngineError = !(headers["Date"] ?? headers["date"] ?? headers["Content-Type"] ?? headers["content-type"]);
+
+        if (body.errors || body.body?.error || isHiddenEngineError) {
+            meta.logger.error("ScrapingBee threw an error", { body: body.body?.error ?? body.errors ?? body.body ?? body });
+            throw new EngineError("Engine error #34", { cause: { body, statusCode: response.status } });
+        }
+
+        if (typeof body.body !== "string") {
+            meta.logger.error("ScrapingBee: Body is not string??", { body });
+            throw new EngineError("Engine error #35", { cause: { body, statusCode: response.status } });
+        }
+
+        specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithScrapingBee/specialtyScrapeCheck" }), body.headers);
+
+        return {
+            url: body["resolved-url"] ?? meta.url,
+
+            html: body.body,
+            error: response.status >= 300 ? response.statusText : undefined,
+            statusCode: response.status,
+            ...(body.screenshot ? ({
+                screenshot: `data:image/png;base64,${body.screenshot}`,
+            }) : {}),
+        };
+    };
+}