From d2f7031069536e0adb07eadc2c0a123b83d5d3ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Mon, 23 Sep 2024 18:33:10 +0200
Subject: [PATCH] fix(WebScraper): fatal error handler triggering for 404s

---
 apps/api/src/scraper/WebScraper/single_url.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts
index 93fb0ce7..d0b7269f 100644
--- a/apps/api/src/scraper/WebScraper/single_url.ts
+++ b/apps/api/src/scraper/WebScraper/single_url.ts
@@ -421,7 +421,9 @@ export async function scrapSingleUrl(
       // }
     }
 
-    if (!text) {
+    // NOTE: This exception for status codes may only work with fire-engine. In lieu of better error management,
+    // it's the best we can do. - mogery
+    if (!text && !Object.values(errors).some(x => x.startsWith("Request failed with status code "))) {
       throw new Error(`All scraping methods failed for URL: ${urlToScrap}`);
     }