fix(queue-worker): finish crawl if all addable URLs were already locked (#1582)
This commit is contained in:
@@ -239,11 +239,15 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
|
|||||||
);
|
);
|
||||||
await addScrapeJobs(lockedJobs);
|
await addScrapeJobs(lockedJobs);
|
||||||
|
|
||||||
logger.info("Added jobs, not going for the full finish", {
|
if (lockedJobs.length > 0) {
|
||||||
lockedJobs: lockedJobs.length,
|
logger.info("Added jobs, not going for the full finish", {
|
||||||
});
|
lockedJobs: lockedJobs.length,
|
||||||
|
});
|
||||||
|
|
||||||
return;
|
return;
|
||||||
|
} else {
|
||||||
|
logger.info("No jobs added (all discovered URLs were locked), finishing crawl");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user