From e97ee4a4be9b856707cae547eedd6d9f015d94e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Sun, 15 Dec 2024 22:33:36 +0100
Subject: [PATCH] fix(WebScraper/tryGetSitemap): deduplicate sitemap links list

---
 apps/api/src/scraper/WebScraper/crawler.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 19b0b5b4..2e47d352 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -210,7 +210,7 @@ export class WebCrawler {
     }
     if (sitemapLinks.length > 0) {
       let filteredLinks = this.filterLinks(
-        sitemapLinks,
+        [...new Set(sitemapLinks)],
         this.limit,
         this.maxCrawledDepth,
         fromMap,