Nick: fixed prettier

This commit is contained in:
Nicolas
2024-12-11 19:46:11 -03:00
parent e5fe9e1534
commit 00335e2ba9
134 changed files with 9565 additions and 7108 deletions
+63 -32
View File
@@ -3,7 +3,13 @@ import { getScrapeQueue } from "./queue-service";
import { v4 as uuidv4 } from "uuid";
import { WebScraperOptions } from "../types";
import * as Sentry from "@sentry/node";
import { cleanOldConcurrencyLimitEntries, getConcurrencyLimitActiveJobs, getConcurrencyLimitMax, pushConcurrencyLimitActiveJob, pushConcurrencyLimitedJob } from "../lib/concurrency-limit";
import {
cleanOldConcurrencyLimitEntries,
getConcurrencyLimitActiveJobs,
getConcurrencyLimitMax,
pushConcurrencyLimitActiveJob,
pushConcurrencyLimitedJob
} from "../lib/concurrency-limit";
async function addScrapeJobRaw(
webScraperOptions: any,
@@ -13,11 +19,17 @@ async function addScrapeJobRaw(
) {
let concurrencyLimited = false;
if (webScraperOptions && webScraperOptions.team_id && webScraperOptions.plan) {
if (
webScraperOptions &&
webScraperOptions.team_id &&
webScraperOptions.plan
) {
const now = Date.now();
const limit = await getConcurrencyLimitMax(webScraperOptions.plan);
cleanOldConcurrencyLimitEntries(webScraperOptions.team_id, now);
concurrencyLimited = (await getConcurrencyLimitActiveJobs(webScraperOptions.team_id, now)).length >= limit;
concurrencyLimited =
(await getConcurrencyLimitActiveJobs(webScraperOptions.team_id, now))
.length >= limit;
}
if (concurrencyLimited) {
@@ -27,19 +39,23 @@ async function addScrapeJobRaw(
opts: {
...options,
priority: jobPriority,
jobId: jobId,
jobId: jobId
},
priority: jobPriority,
priority: jobPriority
});
} else {
if (webScraperOptions && webScraperOptions.team_id && webScraperOptions.plan) {
if (
webScraperOptions &&
webScraperOptions.team_id &&
webScraperOptions.plan
) {
await pushConcurrencyLimitActiveJob(webScraperOptions.team_id, jobId);
}
await getScrapeQueue().add(jobId, webScraperOptions, {
...options,
priority: jobPriority,
jobId,
jobId
});
}
}
@@ -52,24 +68,32 @@ export async function addScrapeJob(
) {
if (Sentry.isInitialized()) {
const size = JSON.stringify(webScraperOptions).length;
return await Sentry.startSpan({
name: "Add scrape job",
op: "queue.publish",
attributes: {
"messaging.message.id": jobId,
"messaging.destination.name": getScrapeQueue().name,
"messaging.message.body.size": size,
return await Sentry.startSpan(
{
name: "Add scrape job",
op: "queue.publish",
attributes: {
"messaging.message.id": jobId,
"messaging.destination.name": getScrapeQueue().name,
"messaging.message.body.size": size
}
},
}, async (span) => {
await addScrapeJobRaw({
...webScraperOptions,
sentry: {
trace: Sentry.spanToTraceHeader(span),
baggage: Sentry.spanToBaggageHeader(span),
size,
},
}, options, jobId, jobPriority);
});
async (span) => {
await addScrapeJobRaw(
{
...webScraperOptions,
sentry: {
trace: Sentry.spanToTraceHeader(span),
baggage: Sentry.spanToBaggageHeader(span),
size
}
},
options,
jobId,
jobPriority
);
}
);
} else {
await addScrapeJobRaw(webScraperOptions, options, jobId, jobPriority);
}
@@ -77,18 +101,25 @@ export async function addScrapeJob(
export async function addScrapeJobs(
jobs: {
data: WebScraperOptions,
data: WebScraperOptions;
opts: {
jobId: string,
priority: number,
},
}[],
jobId: string;
priority: number;
};
}[]
) {
// TODO: better
await Promise.all(jobs.map(job => addScrapeJob(job.data, job.opts, job.opts.jobId, job.opts.priority)));
await Promise.all(
jobs.map((job) =>
addScrapeJob(job.data, job.opts, job.opts.jobId, job.opts.priority)
)
);
}
export function waitForJob<T = unknown>(jobId: string, timeout: number): Promise<T> {
export function waitForJob<T = unknown>(
jobId: string,
timeout: number
): Promise<T> {
return new Promise((resolve, reject) => {
const start = Date.now();
const int = setInterval(async () => {
@@ -110,5 +141,5 @@ export function waitForJob<T = unknown>(jobId: string, timeout: number): Promise
}
}
}, 250);
})
});
}