Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(crawler): treat XML files as sitemaps (temporarily)

Browse Source
This commit is contained in:
Gergő Móricz
2024-11-15 20:09:20 +01:00
parent ca2e33db0a
commit 350d00d27a
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/crawler.ts
+1 -1
View File
@@ -362,7 +362,7 @@ export class WebCrawler {
}; };
const sitemapUrl = url.endsWith("/sitemap.xml") const sitemapUrl = url.endsWith(".xml")
? url ? url
: `${url}/sitemap.xml`; : `${url}/sitemap.xml`;