feat(batchScrape): handle timeout

apps/api/src/services/queue-worker.ts

@@ -276,10 +276,18 @@ async function processJob(job: Job & { id: string }, token: string) {
     });
     const start = Date.now();
 
-    const pipeline = await startWebScraperPipeline({
+    const pipeline = await Promise.race([
+      startWebScraperPipeline({
         job,
         token,
-    });
+      }),
+      ...(job.data.scrapeOptions.timeout !== undefined ? [
+        (async () => {
+          await sleep(job.data.scrapeOptions.timeout);
+          throw new Error("timeout")
+        })(),
+      ] : [])
+    ]);
 
     if (!pipeline.success) {
       // TODO: let's Not do this
@@ -486,6 +494,9 @@ async function processJob(job: Job & { id: string }, token: string) {
     logger.info(`🐂 Job done ${job.id}`);
     return data;
   } catch (error) {
+    const isEarlyTimeout = error instanceof Error && error.message === "timeout";
+
+    if (!isEarlyTimeout) {
       logger.error(`🐂 Job errored ${job.id} - ${error}`);
 
       Sentry.captureException(error, {
@@ -493,6 +504,9 @@ async function processJob(job: Job & { id: string }, token: string) {
           job: job.id,
         },
       });
+    } else {
+      logger.error(`🐂 Job timed out ${job.id}`);
+    }
 
     if (error instanceof CustomError) {
       // Here we handle the error, then save the failed job