From 5992c57158182a7ed93cebc1584cb589b73c3b07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Fri, 17 Jan 2025 17:07:44 +0100
Subject: [PATCH] fix(crawler): bad urls from sitemap

---
 apps/api/src/scraper/WebScraper/crawler.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 1cca7a18..a6645152 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -228,7 +228,7 @@ export class WebCrawler {
         );
         leftOfLimit -= filteredLinks.length;
         let uniqueURLs: string[] = [];
-        for (const url of urls) {
+        for (const url of filteredLinks) {
           if (
             await redisConnection.sadd(
               "sitemap:" + this.jobId + ":links",