From ce460a3a5619c4797e89c62d66b3a41140931eba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Tue, 10 Dec 2024 22:33:53 +0100 Subject: [PATCH] fix(v1/crawl/status): completed more than total if some scrape jobs fail or are discarded --- apps/api/src/lib/crawl-redis.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts index 5cc4d6a4..842f6ebf 100644 --- a/apps/api/src/lib/crawl-redis.ts +++ b/apps/api/src/lib/crawl-redis.ts @@ -53,12 +53,15 @@ export async function addCrawlJobs(id: string, job_ids: string[]) { await redisConnection.expire("crawl:" + id + ":jobs", 24 * 60 * 60, "NX"); } -export async function addCrawlJobDone(id: string, job_id: string) { +export async function addCrawlJobDone(id: string, job_id: string, success: boolean) { _logger.debug("Adding done crawl job to Redis...", { jobId: job_id, module: "crawl-redis", method: "addCrawlJobDone", crawlId: id }); await redisConnection.sadd("crawl:" + id + ":jobs_done", job_id); - await redisConnection.rpush("crawl:" + id + ":jobs_done_ordered", job_id); await redisConnection.expire("crawl:" + id + ":jobs_done", 24 * 60 * 60, "NX"); - await redisConnection.expire("crawl:" + id + ":jobs_done_ordered", 24 * 60 * 60, "NX"); + + if (success) { + await redisConnection.rpush("crawl:" + id + ":jobs_done_ordered", job_id); + await redisConnection.expire("crawl:" + id + ":jobs_done_ordered", 24 * 60 * 60, "NX"); + } } export async function getDoneJobsOrderedLength(id: string): Promise {