Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Nick: sitemap fetch only below threshold for /map

Browse Source
This commit is contained in:
Nicolas
2025-01-21 16:28:57 -03:00
parent c7b219169b
commit ac0d10c451
1 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/controllers/v1/map.ts
+9 -5
View File
@@ -155,15 +155,19 @@ export async function getMapResults({
await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours
} }
// Parallelize sitemap fetch with serper search and sitemap-index // Parallelize sitemap index query with search results
const [_, sitemapIndexUrls, ...searchResults] = await Promise.all([ const [sitemapIndexUrls, ...searchResults] = await Promise.all([
ignoreSitemap ? null : crawler.tryGetSitemap(urls => {
links.push(...urls);
}, true, false, 30000),
querySitemapIndex(url), querySitemapIndex(url),
...(cachedResult ? [] : pagePromises), ...(cachedResult ? [] : pagePromises),
]); ]);
// Only query sitemap if index has less than 100 links
if (!ignoreSitemap && sitemapIndexUrls.length < 100) {
await crawler.tryGetSitemap(urls => {
links.push(...urls);
}, true, false, 30000);
}
if (!cachedResult) { if (!cachedResult) {
allResults = searchResults; allResults = searchResults;
} }