Nick: fixed prettier
This commit is contained in:
@@ -6,47 +6,61 @@ import { Engine } from "../scraper/scrapeURL/engines";
|
||||
configDotenv();
|
||||
|
||||
export type ScrapeErrorEvent = {
|
||||
type: "error",
|
||||
message: string,
|
||||
stack?: string,
|
||||
}
|
||||
type: "error";
|
||||
message: string;
|
||||
stack?: string;
|
||||
};
|
||||
|
||||
export type ScrapeScrapeEvent = {
|
||||
type: "scrape",
|
||||
url: string,
|
||||
worker?: string,
|
||||
method: Engine,
|
||||
type: "scrape";
|
||||
url: string;
|
||||
worker?: string;
|
||||
method: Engine;
|
||||
result: null | {
|
||||
success: boolean,
|
||||
response_code?: number,
|
||||
response_size?: number,
|
||||
error?: string | object,
|
||||
success: boolean;
|
||||
response_code?: number;
|
||||
response_size?: number;
|
||||
error?: string | object;
|
||||
// proxy?: string,
|
||||
time_taken: number,
|
||||
},
|
||||
}
|
||||
time_taken: number;
|
||||
};
|
||||
};
|
||||
|
||||
export type ScrapeQueueEvent = {
|
||||
type: "queue",
|
||||
event: "waiting" | "active" | "completed" | "paused" | "resumed" | "removed" | "failed",
|
||||
worker?: string,
|
||||
}
|
||||
type: "queue";
|
||||
event:
|
||||
| "waiting"
|
||||
| "active"
|
||||
| "completed"
|
||||
| "paused"
|
||||
| "resumed"
|
||||
| "removed"
|
||||
| "failed";
|
||||
worker?: string;
|
||||
};
|
||||
|
||||
export type ScrapeEvent = ScrapeErrorEvent | ScrapeScrapeEvent | ScrapeQueueEvent;
|
||||
export type ScrapeEvent =
|
||||
| ScrapeErrorEvent
|
||||
| ScrapeScrapeEvent
|
||||
| ScrapeQueueEvent;
|
||||
|
||||
export class ScrapeEvents {
|
||||
static async insert(jobId: string, content: ScrapeEvent) {
|
||||
if (jobId === "TEST") return null;
|
||||
|
||||
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
|
||||
|
||||
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
|
||||
if (useDbAuthentication) {
|
||||
try {
|
||||
const result = await supabase.from("scrape_events").insert({
|
||||
job_id: jobId,
|
||||
type: content.type,
|
||||
content: content,
|
||||
// created_at
|
||||
}).select().single();
|
||||
const result = await supabase
|
||||
.from("scrape_events")
|
||||
.insert({
|
||||
job_id: jobId,
|
||||
type: content.type,
|
||||
content: content
|
||||
// created_at
|
||||
})
|
||||
.select()
|
||||
.single();
|
||||
return (result.data as any).id;
|
||||
} catch (error) {
|
||||
// logger.error(`Error inserting scrape event: ${error}`);
|
||||
@@ -57,17 +71,25 @@ export class ScrapeEvents {
|
||||
return null;
|
||||
}
|
||||
|
||||
static async updateScrapeResult(logId: number | null, result: ScrapeScrapeEvent["result"]) {
|
||||
static async updateScrapeResult(
|
||||
logId: number | null,
|
||||
result: ScrapeScrapeEvent["result"]
|
||||
) {
|
||||
if (logId === null) return;
|
||||
|
||||
try {
|
||||
const previousLog = (await supabase.from("scrape_events").select().eq("id", logId).single()).data as any;
|
||||
await supabase.from("scrape_events").update({
|
||||
content: {
|
||||
...previousLog.content,
|
||||
result,
|
||||
}
|
||||
}).eq("id", logId);
|
||||
const previousLog = (
|
||||
await supabase.from("scrape_events").select().eq("id", logId).single()
|
||||
).data as any;
|
||||
await supabase
|
||||
.from("scrape_events")
|
||||
.update({
|
||||
content: {
|
||||
...previousLog.content,
|
||||
result
|
||||
}
|
||||
})
|
||||
.eq("id", logId);
|
||||
} catch (error) {
|
||||
logger.error(`Error updating scrape result: ${error}`);
|
||||
}
|
||||
@@ -78,7 +100,7 @@ export class ScrapeEvents {
|
||||
await this.insert(((job as any).id ? (job as any).id : job) as string, {
|
||||
type: "queue",
|
||||
event,
|
||||
worker: process.env.FLY_MACHINE_ID,
|
||||
worker: process.env.FLY_MACHINE_ID
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(`Error logging job event: ${error}`);
|
||||
|
||||
Reference in New Issue
Block a user