feat(crawl): add parameter to treat differing query parameters as different URLs (#892)

* add parameter to crawleroptions * add code to make it work
2024-11-11 21:36:22 +01:00
parent 5cb46dc494
commit a8dc75f762
7 changed files with 19 additions and 16 deletions
@@ -133,6 +133,7 @@ export async function crawlController(

    await lockURLs(
      id,
+      sc,
      jobs.map((x) => x.data.url)
    );
    await addCrawlJobs(