Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(crawl-redis/addCrawlJobDone): further ensure that completed doesn't go over total

Browse Source
This commit is contained in:
Gergő Móricz
2024-12-15 16:29:09 +01:00
parent b4a5e1a6e9
commit 98f27b0acc
1 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/lib/crawl-redis.ts
+5 -1
View File
@@ -92,13 +92,17 @@ export async function addCrawlJobDone(
if (success) { if (success) {
await redisConnection.rpush("crawl:" + id + ":jobs_done_ordered", job_id); await redisConnection.rpush("crawl:" + id + ":jobs_done_ordered", job_id);
} else {
// in case it's already been pushed, make sure it's removed
await redisConnection.lrem("crawl:" + id + ":jobs_done_ordered", -1, job_id);
}
await redisConnection.expire( await redisConnection.expire(
"crawl:" + id + ":jobs_done_ordered", "crawl:" + id + ":jobs_done_ordered",
24 * 60 * 60, 24 * 60 * 60,
"NX", "NX",
); );
} }
}
export async function getDoneJobsOrderedLength(id: string): Promise<number> { export async function getDoneJobsOrderedLength(id: string): Promise<number> {
return await redisConnection.llen("crawl:" + id + ":jobs_done_ordered"); return await redisConnection.llen("crawl:" + id + ":jobs_done_ordered");