Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'v1-webscraper' of https://github.com/mendableai/firecrawl into v1-webscraper

Browse Source
This commit is contained in:
Nicolas
2024-08-28 14:09:20 -03:00
parent 08387c06f8 ef2d8d012b
commit 8fbff28009
2 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/index.ts
+2 -1
View File
@@ -582,8 +582,9 @@ export class WebScraperDataProvider {
this.pageOptions = { this.pageOptions = {
onlyMainContent: options.pageOptions?.onlyMainContent ?? false, onlyMainContent: options.pageOptions?.onlyMainContent ?? false,
includeHtml: options.pageOptions?.includeHtml ?? false, includeHtml: options.pageOptions?.includeHtml ?? false,
replaceAllPathsWithAbsolutePaths: options.pageOptions?.replaceAllPathsWithAbsolutePaths ?? false, replaceAllPathsWithAbsolutePaths: options.pageOptions?.replaceAllPathsWithAbsolutePaths ?? true,
parsePDF: options.pageOptions?.parsePDF ?? true, parsePDF: options.pageOptions?.parsePDF ?? true,
onlyIncludeTags: options.pageOptions?.onlyIncludeTags ?? [],
removeTags: options.pageOptions?.removeTags ?? [], removeTags: options.pageOptions?.removeTags ?? [],
includeMarkdown: options.pageOptions?.includeMarkdown ?? true, includeMarkdown: options.pageOptions?.includeMarkdown ?? true,
includeRawHtml: options.pageOptions?.includeRawHtml ?? false, includeRawHtml: options.pageOptions?.includeRawHtml ?? false,
apps/api/src/scraper/WebScraper/single_url.ts
+2 -1
View File
@@ -138,9 +138,10 @@ export async function scrapSingleUrl(
fullPageScreenshot: pageOptions.fullPageScreenshot ?? false, fullPageScreenshot: pageOptions.fullPageScreenshot ?? false,
headers: pageOptions.headers ?? undefined, headers: pageOptions.headers ?? undefined,
includeLinks: pageOptions.includeLinks ?? true, includeLinks: pageOptions.includeLinks ?? true,
replaceAllPathsWithAbsolutePaths: pageOptions.replaceAllPathsWithAbsolutePaths ?? false, replaceAllPathsWithAbsolutePaths: pageOptions.replaceAllPathsWithAbsolutePaths ?? true,
parsePDF: pageOptions.parsePDF ?? true, parsePDF: pageOptions.parsePDF ?? true,
removeTags: pageOptions.removeTags ?? [], removeTags: pageOptions.removeTags ?? [],
onlyIncludeTags: pageOptions.onlyIncludeTags ?? [],
} }
if (extractorOptions) { if (extractorOptions) {