Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update sitemap.ts

Browse Source
This commit is contained in:
Nicolas
2024-09-05 17:52:27 -03:00
parent 554a05068c
commit f5b84e15e1
1 changed files with 8 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/sitemap.ts
+8 -10
View File
@@ -36,17 +36,15 @@ export async function getLinksFromSitemap(
const root = parsed.urlset || parsed.sitemapindex; const root = parsed.urlset || parsed.sitemapindex;
if (root && root.sitemap) { if (root && root.sitemap) {
for (const sitemap of root.sitemap) { const sitemapPromises = root.sitemap
if (sitemap.loc && sitemap.loc.length > 0) { .filter(sitemap => sitemap.loc && sitemap.loc.length > 0)
await getLinksFromSitemap({ sitemapUrl: sitemap.loc[0], allUrls, mode }); .map(sitemap => getLinksFromSitemap({ sitemapUrl: sitemap.loc[0], allUrls, mode }));
} await Promise.all(sitemapPromises);
}
} else if (root && root.url) { } else if (root && root.url) {
for (const url of root.url) { const validUrls = root.url
if (url.loc && url.loc.length > 0 && !WebCrawler.prototype.isFile(url.loc[0])) { .filter(url => url.loc && url.loc.length > 0 && !WebCrawler.prototype.isFile(url.loc[0]))
allUrls.push(url.loc[0]); .map(url => url.loc[0]);
} allUrls.push(...validUrls);
}
} }
} catch (error) { } catch (error) {
Logger.debug(`Error processing sitemapUrl: ${sitemapUrl} | Error: ${error.message}`); Logger.debug(`Error processing sitemapUrl: ${sitemapUrl} | Error: ${error.message}`);