concurrency limit fix PoC II.

2024-10-25 20:21:12 +02:00
parent dbcf2d7ff6
commit 4468a49aee
10 changed files with 162 additions and 102 deletions
@@ -137,7 +137,8 @@ export async function crawlController(
    await getScrapeQueue().addBulk(jobs);
  } else {
    await lockURL(id, sc, req.body.url);
-    const job = await addScrapeJob(
+    const jobId = uuidv4();
+    await addScrapeJob(
      {
        url: req.body.url,
        mode: "single_urls",
@@ -152,9 +153,10 @@ export async function crawlController(
      },
      {
        priority: 15,
-      }
+      },
+      jobId,
    );
-    await addCrawlJob(id, job.id);
+    await addCrawlJob(id, jobId);
  }

  if(req.body.webhook) {