fix(crawler): sitemaps poisoning crawls with unrelated links (#1334)

2025-03-13 19:31:05 +01:00
parent da6b7505f3
commit c7ae50d2d0
1 changed files with 1 additions and 1 deletions
@@ -271,7 +271,7 @@ export class WebCrawler {
        return urlsHandler(urls);
      } else {
        let filteredLinks = this.filterLinks(
-          [...new Set(urls)],
+          [...new Set(urls)].filter(x => this.filterURL(x, this.initialUrl) !== null),
          leftOfLimit,
          this.maxCrawledDepth,
          fromMap,