Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #449 from mendableai/bugfix/malformed-url-sitemap

Browse Source 
Added regex for links in sitemap
This commit is contained in:
Nicolas
2024-07-24 20:37:35 -04:00
committed by GitHub
parent 6ad7e24403 5e728c1a4d
commit 2c1221750b
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/crawler.ts
+1 -1
View File
@@ -64,7 +64,7 @@ export class WebCrawler {
private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] { private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] {
return sitemapLinks return sitemapLinks
.filter((link) => { .filter((link) => {
const url = new URL(link); const url = new URL(link.trim(), this.baseUrl);
const path = url.pathname; const path = url.pathname;
const depth = getURLDepth(url.toString()); const depth = getURLDepth(url.toString());