diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts index dfd17c63..d7ec2a83 100644 --- a/apps/api/src/lib/entities.ts +++ b/apps/api/src/lib/entities.ts @@ -28,7 +28,7 @@ export type PageOptions = { onlyIncludeTags?: string | string[]; includeLinks?: boolean; useFastMode?: boolean; // beta - disableJSDom?: boolean; // beta + disableJsDom?: boolean; // beta atsv?: boolean; // beta }; diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts index 8bd7d493..2f7efa47 100644 --- a/apps/api/src/scraper/WebScraper/index.ts +++ b/apps/api/src/scraper/WebScraper/index.ts @@ -590,7 +590,7 @@ export class WebScraperDataProvider { fullPageScreenshot: options.pageOptions?.fullPageScreenshot ?? false, screenshot: options.pageOptions?.screenshot ?? false, useFastMode: options.pageOptions?.useFastMode ?? false, - disableJSDom: options.pageOptions?.disableJSDom ?? false, + disableJsDom: options.pageOptions?.disableJsDom ?? false, atsv: options.pageOptions?.atsv ?? false }; this.extractorOptions = options.extractorOptions ?? { mode: "markdown" }; diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index aa86ad5e..d5f764b5 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -104,12 +104,13 @@ export async function scrapWithFireEngine({ screenshot: screenshotParam, fullPageScreenshot: fullPageScreenshotParam, headers: headers, - pageOptions: pageOptions, disableJsDom: pageOptions?.disableJsDom ?? false, priority, engine, instantReturn: true, ...fireEngineOptionsParam, + atsv: pageOptions?.atsv ?? false, + scrollXPaths: pageOptions?.scrollXPaths ?? [], }, { headers: { diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index f39f045f..8bafd203 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -147,7 +147,7 @@ export async function scrapSingleUrl( removeTags: pageOptions.removeTags ?? [], onlyIncludeTags: pageOptions.onlyIncludeTags ?? [], useFastMode: pageOptions.useFastMode ?? false, - disableJSDom: pageOptions.disableJSDom ?? false, + disableJsDom: pageOptions.disableJsDom ?? false, atsv: pageOptions.atsv ?? false } @@ -203,6 +203,7 @@ export async function scrapSingleUrl( fireEngineOptions: { engine: engine, atsv: pageOptions.atsv, + disableJsDom: pageOptions.disableJsDom, }, priority, teamId,