Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(WebScraper/tryGetSitemap): deduplicate sitemap links list

Browse Source
This commit is contained in:
Gergő Móricz
2024-12-15 22:33:36 +01:00
parent 37f58efe45
commit e97ee4a4be
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/crawler.ts
+1 -1
View File
@@ -210,7 +210,7 @@ export class WebCrawler {
} }
if (sitemapLinks.length > 0) { if (sitemapLinks.length > 0) {
let filteredLinks = this.filterLinks( let filteredLinks = this.filterLinks(
sitemapLinks, [...new Set(sitemapLinks)],
this.limit, this.limit,
this.maxCrawledDepth, this.maxCrawledDepth,
fromMap, fromMap,