Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #491 from mendableai/bugfix/issue-477

Browse Source 
[Bug] Nested sitemaps
This commit is contained in:
Nicolas
2024-08-02 13:03:28 -04:00
committed by GitHub
parent af68b7a785 4051630632
commit 893113a39e
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/crawler.ts
+1 -1
View File
@@ -476,7 +476,7 @@ export class WebCrawler {
try { try {
const response = await axios.get(baseUrlSitemap, { timeout: axiosTimeout }); const response = await axios.get(baseUrlSitemap, { timeout: axiosTimeout });
if (response.status === 200) { if (response.status === 200) {
sitemapLinks = await getLinksFromSitemap({ sitemapUrl: baseUrlSitemap }); sitemapLinks = await getLinksFromSitemap({ sitemapUrl: baseUrlSitemap, mode: 'fire-engine' });
} }
} catch (error) { } catch (error) {
Logger.debug(`Failed to fetch sitemap from ${baseUrlSitemap}: ${error}`); Logger.debug(`Failed to fetch sitemap from ${baseUrlSitemap}: ${error}`);
apps/api/src/scraper/WebScraper/sitemap.ts
+1 -1
View File
@@ -19,7 +19,7 @@ export async function getLinksFromSitemap(
try { try {
let content: string; let content: string;
try { try {
if (mode === 'axios') { if (mode === 'axios' || process.env.FIRE_ENGINE_BETA_URL === '') {
const response = await axios.get(sitemapUrl, { timeout: axiosTimeout }); const response = await axios.get(sitemapUrl, { timeout: axiosTimeout });
content = response.data; content = response.data;
} else if (mode === 'fire-engine') { } else if (mode === 'fire-engine') {