Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added regex for links in sitemap

Browse Source
This commit is contained in:
rafaelsideguide
2024-07-23 09:07:23 -03:00
parent 252bc09ee2
commit a684bd3c5d
1 changed files with 8 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/crawler.ts
+8
View File
@@ -64,6 +64,14 @@ export class WebCrawler {
private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] { private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] {
return sitemapLinks return sitemapLinks
.filter((link) => { .filter((link) => {
// if link is not a complete url, add the base url
link = link.trim();
const isCompleteUrl = new RegExp('^(?:[a-z+]+:)?//', 'i');
if (!isCompleteUrl.test(link)){
link = this.baseUrl + link;
}
const url = new URL(link); const url = new URL(link);
const path = url.pathname; const path = url.pathname;