diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 19b0b5b4..2e47d352 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -210,7 +210,7 @@ export class WebCrawler {
     }
     if (sitemapLinks.length > 0) {
       let filteredLinks = this.filterLinks(
-        sitemapLinks,
+        [...new Set(sitemapLinks)],
         this.limit,
         this.maxCrawledDepth,
         fromMap,