Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(crawler/rust): dedupe

Browse Source
This commit is contained in:
Móricz Gergő
2025-01-30 08:16:51 +01:00
parent c88176a596
commit a7eb2f7c6a
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/crawler.ts
+2 -2
View File
@@ -401,13 +401,13 @@ export class WebCrawler {
public async extractLinksFromHTML(html: string, url: string) { public async extractLinksFromHTML(html: string, url: string) {
try { try {
return (await this.extractLinksFromHTMLRust(html, url)).map(x => { return [...new Set((await this.extractLinksFromHTMLRust(html, url)).map(x => {
try { try {
return new URL(x, url).href return new URL(x, url).href
} catch (e) { } catch (e) {
return null; return null;
} }
}).filter(x => x !== null) as string[]; }).filter(x => x !== null) as string[])];
} catch (error) { } catch (error) {
this.logger.error("Failed to call html-transformer! Falling back to cheerio...", { this.logger.error("Failed to call html-transformer! Falling back to cheerio...", {
error, error,