testing crawl with new.abb.com case

many unnecessary console.logs for tracing the code execution
This commit is contained in:
rafaelsideguide
2024-06-24 16:25:07 -03:00
parent 3c7b7e7242
commit 21d29de819
12 changed files with 128 additions and 35 deletions
@@ -4,7 +4,7 @@ export async function attemptScrapWithRequests(
urlToScrap: string
): Promise<string | null> {
try {
const response = await axios.get(urlToScrap);
const response = await axios.get(urlToScrap, { timeout: 15000 });
if (!response.data) {
console.log("Failed normal requests as well");