diff --git a/apps/api/src/lib/scrape-events.ts b/apps/api/src/lib/scrape-events.ts index d7958d14..9b050c30 100644 --- a/apps/api/src/lib/scrape-events.ts +++ b/apps/api/src/lib/scrape-events.ts @@ -9,10 +9,13 @@ export type ScrapeErrorEvent = { export type ScrapeScrapeEvent = { type: "scrape", + url: string, + worker?: string, method: (typeof baseScrapers)[number], result: null | { success: boolean, response_code?: number, + response_size?: number, error?: string | object, // proxy?: string, time_taken: number, diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 61ee89af..4b428d35 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -150,6 +150,8 @@ export async function scrapSingleUrl( const timer = Date.now(); const logInsertPromise = ScrapeEvents.insert(jobId, { type: "scrape", + url, + worker: process.env.FLY_MACHINE_ID, method, result: null, }); @@ -267,6 +269,7 @@ export async function scrapSingleUrl( const insertedLogId = await logInsertPromise; ScrapeEvents.updateScrapeResult(insertedLogId, { + response_size: scraperResponse.text.length, success: !scraperResponse.metadata.pageError && !!text, error: scraperResponse.metadata.pageError, response_code: scraperResponse.metadata.pageStatusCode,