fix(scrapeURL/pdf): retry

apps/api/src/scraper/scrapeURL/engines/pdf/index.ts

@@ -62,6 +62,8 @@ async function scrapePDFWithLlamaParse(meta: Meta, tempFilePath: string): Promis
         schema: z.object({
             markdown: z.string(),
         }),
+        tryCount: 16,
+        tryCooldown: 250,
     });
     
     return {

apps/api/src/scraper/scrapeURL/lib/fetch.ts

@@ -15,6 +15,7 @@ export type RobustFetchParams<Schema extends z.Schema<any>> = {
     ignoreFailure?: boolean;
     requestId?: string;
     tryCount?: number;
+    tryCooldown?: number;
 };
 
 export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer<Schema>>({
@@ -28,8 +29,9 @@ export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer
     ignoreFailure = false,
     requestId = uuid(),
     tryCount = 1,
+    tryCooldown,
 }: RobustFetchParams<Schema>): Promise<Output> {
-    const params = { url, logger, method, body, headers, schema, ignoreResponse, ignoreFailure, tryCount };
+    const params = { url, logger, method, body, headers, schema, ignoreResponse, ignoreFailure, tryCount, tryCooldown };
 
     let request: Response;
     try {
@@ -86,6 +88,9 @@ export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer
     if (request.status >= 300) {
         if (tryCount > 1) {
             logger.debug("Request sent failure status, trying " + (tryCount - 1) + " more times", { params, request, response, requestId });
+            if (tryCooldown !== undefined) {
+                await new Promise((resolve) => setTimeout(() => resolve(null), tryCooldown));
+            }
             return await robustFetch({
                 ...params,
                 requestId,