feat(monitoring/scrape): include url, worker, response_size

apps/api/src/lib/scrape-events.ts

@@ -9,10 +9,13 @@ export type ScrapeErrorEvent = {
 
 export type ScrapeScrapeEvent = {
   type: "scrape",
+  url: string,
+  worker?: string,
   method: (typeof baseScrapers)[number],
   result: null | {
     success: boolean,
     response_code?: number,
+    response_size?: number,
     error?: string | object,
     // proxy?: string,
     time_taken: number,

apps/api/src/scraper/WebScraper/single_url.ts

@@ -150,6 +150,8 @@ export async function scrapSingleUrl(
     const timer = Date.now();
     const logInsertPromise = ScrapeEvents.insert(jobId, {
       type: "scrape",
+      url,
+      worker: process.env.FLY_MACHINE_ID,
       method,
       result: null,
     });
@@ -267,6 +269,7 @@ export async function scrapSingleUrl(
 
     const insertedLogId = await logInsertPromise;
     ScrapeEvents.updateScrapeResult(insertedLogId, {
+      response_size: scraperResponse.text.length,
       success: !scraperResponse.metadata.pageError && !!text,
       error: scraperResponse.metadata.pageError,
       response_code: scraperResponse.metadata.pageStatusCode,