fix(scrapeURL/sb): enforce timeout (FIR-980) (#1183)

* fix(scrapeURL/scrapingbee): enforce timeout

* fix(scrapeURL/sb): types

* fix the test

* fixup: remove nix files

apps/api/src/__tests__/snips/scrape.test.ts

@@ -34,7 +34,7 @@ describe("Scrape tests", () => {
     expect(response.body.data.markdown).toBe(
       "this is fake data coming from the mocking system!",
     );
-  });
+  }, 10000);
 
   describe("Ad blocking (f-e dependant)", () => {
     it.concurrent("blocks ads by default", async () => {

apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts

@@ -3,7 +3,7 @@ import { Meta } from "../..";
 import { EngineScrapeResult } from "..";
 import { specialtyScrapeCheck } from "../utils/specialtyHandler";
 import { AxiosError, type AxiosResponse } from "axios";
-import { EngineError } from "../../error";
+import { EngineError, TimeoutError } from "../../error";
 
 const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY!);
 
@@ -17,7 +17,8 @@ export function scrapeURLWithScrapingBee(
     let response: AxiosResponse<any>;
     const timeout = (timeToRun ?? 300000) + meta.options.waitFor;
     try {
-      response = await client.get({
+      response = await Promise.race<AxiosResponse<any>>([
+        client.get({
           url: meta.url,
           params: {
             timeout,
@@ -33,7 +34,9 @@ export function scrapeURLWithScrapingBee(
           headers: {
             "ScrapingService-Request": "TRUE", // this is sent to the page, not to ScrapingBee - mogery
           },
-      });
+        }),
+        new Promise((_, reject) => setTimeout(() => reject(new TimeoutError("ScrapingBee timed out")), timeout + 5000)),
+      ]);
     } catch (error) {
       if (error instanceof AxiosError && error.response !== undefined) {
         response = error.response;