feat(crawl): add parameter to treat differing query parameters as different URLs (#892)

* add parameter to crawleroptions * add code to make it work
2024-11-11 21:36:22 +01:00
parent 5cb46dc494
commit a8dc75f762
7 changed files with 19 additions and 16 deletions
@@ -155,6 +155,7 @@ export interface CrawlParams {
  scrapeOptions?: CrawlScrapeOptions;
  webhook?: string;
  deduplicateSimilarURLs?: boolean;
+  ignoreQueryParameters?: boolean;
 }

 /**