Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(queue-worker): only do redirect handling logic on crawls, not batch scrape

Browse Source
This commit is contained in:
Gergő Móricz
2025-01-04 16:59:35 +01:00
parent d48ddb8820
commit b92a4eb79b
2 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/controllers/v1/batch-scrape.ts
+1 -1
View File
@@ -61,7 +61,7 @@ export async function batchScrapeController(
} }
logger.debug("Batch scrape " + id + " starting", { logger.debug("Batch scrape " + id + " starting", {
urlsLength: urls, urlsLength: urls.length,
appendToId: req.body.appendToId, appendToId: req.body.appendToId,
account: req.account, account: req.account,
}); });
apps/api/src/services/queue-worker.ts
+2 -1
View File
@@ -710,7 +710,8 @@ async function processJob(job: Job & { id: string }, token: string) {
doc.metadata.url !== undefined && doc.metadata.url !== undefined &&
doc.metadata.sourceURL !== undefined && doc.metadata.sourceURL !== undefined &&
normalizeURL(doc.metadata.url, sc) !== normalizeURL(doc.metadata.url, sc) !==
normalizeURL(doc.metadata.sourceURL, sc) normalizeURL(doc.metadata.sourceURL, sc) &&
job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape
) { ) {
const crawler = crawlToCrawler(job.data.crawl_id, sc); const crawler = crawlToCrawler(job.data.crawl_id, sc);
if ( if (