Nick: fixed prettier
This commit is contained in:
@@ -3,7 +3,13 @@ import { getScrapeQueue } from "./queue-service";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { WebScraperOptions } from "../types";
|
||||
import * as Sentry from "@sentry/node";
|
||||
import { cleanOldConcurrencyLimitEntries, getConcurrencyLimitActiveJobs, getConcurrencyLimitMax, pushConcurrencyLimitActiveJob, pushConcurrencyLimitedJob } from "../lib/concurrency-limit";
|
||||
import {
|
||||
cleanOldConcurrencyLimitEntries,
|
||||
getConcurrencyLimitActiveJobs,
|
||||
getConcurrencyLimitMax,
|
||||
pushConcurrencyLimitActiveJob,
|
||||
pushConcurrencyLimitedJob
|
||||
} from "../lib/concurrency-limit";
|
||||
|
||||
async function addScrapeJobRaw(
|
||||
webScraperOptions: any,
|
||||
@@ -13,11 +19,17 @@ async function addScrapeJobRaw(
|
||||
) {
|
||||
let concurrencyLimited = false;
|
||||
|
||||
if (webScraperOptions && webScraperOptions.team_id && webScraperOptions.plan) {
|
||||
if (
|
||||
webScraperOptions &&
|
||||
webScraperOptions.team_id &&
|
||||
webScraperOptions.plan
|
||||
) {
|
||||
const now = Date.now();
|
||||
const limit = await getConcurrencyLimitMax(webScraperOptions.plan);
|
||||
cleanOldConcurrencyLimitEntries(webScraperOptions.team_id, now);
|
||||
concurrencyLimited = (await getConcurrencyLimitActiveJobs(webScraperOptions.team_id, now)).length >= limit;
|
||||
concurrencyLimited =
|
||||
(await getConcurrencyLimitActiveJobs(webScraperOptions.team_id, now))
|
||||
.length >= limit;
|
||||
}
|
||||
|
||||
if (concurrencyLimited) {
|
||||
@@ -27,19 +39,23 @@ async function addScrapeJobRaw(
|
||||
opts: {
|
||||
...options,
|
||||
priority: jobPriority,
|
||||
jobId: jobId,
|
||||
jobId: jobId
|
||||
},
|
||||
priority: jobPriority,
|
||||
priority: jobPriority
|
||||
});
|
||||
} else {
|
||||
if (webScraperOptions && webScraperOptions.team_id && webScraperOptions.plan) {
|
||||
if (
|
||||
webScraperOptions &&
|
||||
webScraperOptions.team_id &&
|
||||
webScraperOptions.plan
|
||||
) {
|
||||
await pushConcurrencyLimitActiveJob(webScraperOptions.team_id, jobId);
|
||||
}
|
||||
|
||||
await getScrapeQueue().add(jobId, webScraperOptions, {
|
||||
...options,
|
||||
priority: jobPriority,
|
||||
jobId,
|
||||
jobId
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -52,24 +68,32 @@ export async function addScrapeJob(
|
||||
) {
|
||||
if (Sentry.isInitialized()) {
|
||||
const size = JSON.stringify(webScraperOptions).length;
|
||||
return await Sentry.startSpan({
|
||||
name: "Add scrape job",
|
||||
op: "queue.publish",
|
||||
attributes: {
|
||||
"messaging.message.id": jobId,
|
||||
"messaging.destination.name": getScrapeQueue().name,
|
||||
"messaging.message.body.size": size,
|
||||
return await Sentry.startSpan(
|
||||
{
|
||||
name: "Add scrape job",
|
||||
op: "queue.publish",
|
||||
attributes: {
|
||||
"messaging.message.id": jobId,
|
||||
"messaging.destination.name": getScrapeQueue().name,
|
||||
"messaging.message.body.size": size
|
||||
}
|
||||
},
|
||||
}, async (span) => {
|
||||
await addScrapeJobRaw({
|
||||
...webScraperOptions,
|
||||
sentry: {
|
||||
trace: Sentry.spanToTraceHeader(span),
|
||||
baggage: Sentry.spanToBaggageHeader(span),
|
||||
size,
|
||||
},
|
||||
}, options, jobId, jobPriority);
|
||||
});
|
||||
async (span) => {
|
||||
await addScrapeJobRaw(
|
||||
{
|
||||
...webScraperOptions,
|
||||
sentry: {
|
||||
trace: Sentry.spanToTraceHeader(span),
|
||||
baggage: Sentry.spanToBaggageHeader(span),
|
||||
size
|
||||
}
|
||||
},
|
||||
options,
|
||||
jobId,
|
||||
jobPriority
|
||||
);
|
||||
}
|
||||
);
|
||||
} else {
|
||||
await addScrapeJobRaw(webScraperOptions, options, jobId, jobPriority);
|
||||
}
|
||||
@@ -77,18 +101,25 @@ export async function addScrapeJob(
|
||||
|
||||
export async function addScrapeJobs(
|
||||
jobs: {
|
||||
data: WebScraperOptions,
|
||||
data: WebScraperOptions;
|
||||
opts: {
|
||||
jobId: string,
|
||||
priority: number,
|
||||
},
|
||||
}[],
|
||||
jobId: string;
|
||||
priority: number;
|
||||
};
|
||||
}[]
|
||||
) {
|
||||
// TODO: better
|
||||
await Promise.all(jobs.map(job => addScrapeJob(job.data, job.opts, job.opts.jobId, job.opts.priority)));
|
||||
await Promise.all(
|
||||
jobs.map((job) =>
|
||||
addScrapeJob(job.data, job.opts, job.opts.jobId, job.opts.priority)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function waitForJob<T = unknown>(jobId: string, timeout: number): Promise<T> {
|
||||
export function waitForJob<T = unknown>(
|
||||
jobId: string,
|
||||
timeout: number
|
||||
): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const start = Date.now();
|
||||
const int = setInterval(async () => {
|
||||
@@ -110,5 +141,5 @@ export function waitForJob<T = unknown>(jobId: string, timeout: number): Promise
|
||||
}
|
||||
}
|
||||
}, 250);
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user