fix(concurrency): proper job timeouting

This commit is contained in:
Móricz Gergő
2025-01-31 11:22:10 +01:00
parent e026917698
commit 8a8d7d645f
5 changed files with 44 additions and 9 deletions
+10 -1
View File
@@ -4,6 +4,7 @@ import { v4 as uuidv4 } from "uuid";
import { NotificationType, PlanType, WebScraperOptions } from "../types";
import * as Sentry from "@sentry/node";
import {
calculateJobTimeToRun,
cleanOldConcurrencyLimitEntries,
getConcurrencyLimitActiveJobs,
getConcurrencyQueueJobsCount,
@@ -43,7 +44,15 @@ export async function _addScrapeJobToBullMQ(
webScraperOptions.team_id &&
webScraperOptions.plan
) {
await pushConcurrencyLimitActiveJob(webScraperOptions.team_id, jobId);
await pushConcurrencyLimitActiveJob(webScraperOptions.team_id, jobId, calculateJobTimeToRun({
id: jobId,
opts: {
...options,
priority: jobPriority,
jobId,
},
data: webScraperOptions,
}));
}
await getScrapeQueue().add(jobId, webScraperOptions, {