feat: scrape event logging to DB

apps/api/src/controllers/crawl.ts

@@ -10,6 +10,7 @@ import { logCrawl } from "../../src/services/logging/crawl_log";
 import { validateIdempotencyKey } from "../../src/services/idempotency/validate";
 import { createIdempotencyKey } from "../../src/services/idempotency/create";
 import { defaultCrawlPageOptions, defaultCrawlerOptions, defaultOrigin } from "../../src/lib/default-values";
+import { v4 as uuidv4 } from "uuid";
 
 export async function crawlController(req: Request, res: Response) {
   try {
@@ -60,10 +61,11 @@ export async function crawlController(req: Request, res: Response) {
     const crawlerOptions = { ...defaultCrawlerOptions, ...req.body.crawlerOptions };
     const pageOptions = { ...defaultCrawlPageOptions, ...req.body.pageOptions };
 
-    if (mode === "single_urls" && !url.includes(",")) {
+    if (mode === "single_urls" && !url.includes(",")) { // NOTE: do we need this?
       try {
         const a = new WebScraperDataProvider();
         await a.setOptions({
+          jobId: uuidv4(),
           mode: "single_urls",
           urls: [url],
           crawlerOptions: { ...crawlerOptions, returnOnlyUrls: true },