From 4d35ad073c86e74840dc0074cdee1e7a5a05ea90 Mon Sep 17 00:00:00 2001
From: Gergo Moricz <mo.geryy@gmail.com>
Date: Wed, 24 Jul 2024 16:43:39 +0200
Subject: [PATCH] feat(monitoring/scrape): include url, worker, response_size

---
 apps/api/src/lib/scrape-events.ts             | 3 +++
 apps/api/src/scraper/WebScraper/single_url.ts | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/apps/api/src/lib/scrape-events.ts b/apps/api/src/lib/scrape-events.ts
index d7958d14..9b050c30 100644
--- a/apps/api/src/lib/scrape-events.ts
+++ b/apps/api/src/lib/scrape-events.ts
@@ -9,10 +9,13 @@ export type ScrapeErrorEvent = {
 
 export type ScrapeScrapeEvent = {
   type: "scrape",
+  url: string,
+  worker?: string,
   method: (typeof baseScrapers)[number],
   result: null | {
     success: boolean,
     response_code?: number,
+    response_size?: number,
     error?: string | object,
     // proxy?: string,
     time_taken: number,
diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts
index 61ee89af..4b428d35 100644
--- a/apps/api/src/scraper/WebScraper/single_url.ts
+++ b/apps/api/src/scraper/WebScraper/single_url.ts
@@ -150,6 +150,8 @@ export async function scrapSingleUrl(
     const timer = Date.now();
     const logInsertPromise = ScrapeEvents.insert(jobId, {
       type: "scrape",
+      url,
+      worker: process.env.FLY_MACHINE_ID,
       method,
       result: null,
     });
@@ -267,6 +269,7 @@ export async function scrapSingleUrl(
 
     const insertedLogId = await logInsertPromise;
     ScrapeEvents.updateScrapeResult(insertedLogId, {
+      response_size: scraperResponse.text.length,
       success: !scraperResponse.metadata.pageError && !!text,
       error: scraperResponse.metadata.pageError,
       response_code: scraperResponse.metadata.pageStatusCode,