feat(sitemap): change sitemap logging

apps/api/src/scraper/WebScraper/crawler.ts

@@ -632,6 +632,10 @@ export class WebCrawler {
       }
     }
 
+    if (this.sitemapsHit.size >= 20) {
+      this.logger.warn("Sitemap limit hit!", { crawlId: this.jobId, url: this.baseUrl });
+    }
+
     return sitemapCount;
   }
 }

apps/api/src/scraper/WebScraper/sitemap.ts

@@ -22,7 +22,6 @@ export async function getLinksFromSitemap(
   sitemapsHit: Set<string>,
 ): Promise<number> {
   if (sitemapsHit.size >= 20) {
-    logger.warn("Sitemap limit hit, not hitting this one.");
     return 0;
   }