2024-07-30 13:27:23 -04:00
|
|
|
import { Job, Queue } from "bullmq";
|
2024-08-13 21:03:24 +02:00
|
|
|
import { getScrapeQueue } from "./queue-service";
|
2024-04-15 17:01:47 -04:00
|
|
|
import { v4 as uuidv4 } from "uuid";
|
|
|
|
|
import { WebScraperOptions } from "../types";
|
|
|
|
|
|
2024-07-30 14:44:13 -04:00
|
|
|
export async function addScrapeJob(
|
|
|
|
|
webScraperOptions: WebScraperOptions,
|
|
|
|
|
options: any = {},
|
|
|
|
|
jobId: string = uuidv4(),
|
|
|
|
|
): Promise<Job> {
|
|
|
|
|
return await getScrapeQueue().add(jobId, webScraperOptions, {
|
2024-08-15 18:58:07 +02:00
|
|
|
priority: webScraperOptions.crawl_id ? 20 : 10,
|
2024-07-30 14:44:13 -04:00
|
|
|
...options,
|
|
|
|
|
jobId,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|