From ac0d10c45171eba19aab95324eaf32781dbb36c9 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 21 Jan 2025 16:28:57 -0300 Subject: [PATCH] Nick: sitemap fetch only below threshold for /map --- apps/api/src/controllers/v1/map.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts index 75e253ba..da12980c 100644 --- a/apps/api/src/controllers/v1/map.ts +++ b/apps/api/src/controllers/v1/map.ts @@ -155,15 +155,19 @@ export async function getMapResults({ await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours } - // Parallelize sitemap fetch with serper search and sitemap-index - const [_, sitemapIndexUrls, ...searchResults] = await Promise.all([ - ignoreSitemap ? null : crawler.tryGetSitemap(urls => { - links.push(...urls); - }, true, false, 30000), + // Parallelize sitemap index query with search results + const [sitemapIndexUrls, ...searchResults] = await Promise.all([ querySitemapIndex(url), ...(cachedResult ? [] : pagePromises), ]); + // Only query sitemap if index has less than 100 links + if (!ignoreSitemap && sitemapIndexUrls.length < 100) { + await crawler.tryGetSitemap(urls => { + links.push(...urls); + }, true, false, 30000); + } + if (!cachedResult) { allResults = searchResults; }