Merge branch 'main' into feat/better-gdrive-pdf-fetch

apps/api/src/scraper/WebScraper/single_url.ts

@@ -46,6 +46,7 @@ export async function scrapWithFireEngine(
   url: string,
   waitFor: number = 0,
   screenshot: boolean = false,
+  pageOptions: { scrollXPaths?: string[] } = {},
   headers?: Record<string, string>,
   options?: any
 ): Promise<FireEngineResponse> {
@@ -68,6 +69,7 @@ export async function scrapWithFireEngine(
         wait: waitParam,
         screenshot: screenshotParam,
         headers: headers,
+        pageOptions: pageOptions
       }),
     });
 
@@ -334,7 +336,7 @@ export async function scrapSingleUrl(
     if (customScraperResult){
       switch (customScraperResult.scraper) {
         case "fire-engine":
-          customScrapedContent  = await scrapWithFireEngine(customScraperResult.url, customScraperResult.wait_after_load)
+          customScrapedContent  = await scrapWithFireEngine(customScraperResult.url, customScraperResult.waitAfterLoad, false, customScraperResult.pageOptions)
         case "pdf":
           customScrapedContent  = { html: await fetchAndProcessPdf(customScraperResult.url), screenshot }
       }