Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix for maxDepth

Browse Source
This commit is contained in:
Eric Ciarla
2024-06-14 19:40:37 -04:00
parent 354712a8a3
commit a6b7197737
3 changed files with 76 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/index.ts
+2 -2
View File
@@ -164,9 +164,9 @@ export class WebScraperDataProvider {
): Promise<Document[]> {
const pathSplits = new URL(this.urls[0]).pathname.split('/');
const baseURLDepth = pathSplits.length - (pathSplits[0].length === 0 && pathSplits[pathSplits.length - 1].length === 0 ? 1 : 0);
const baseURLDepth = pathSplits.length - (pathSplits[0].length === 0 && pathSplits[pathSplits.length - 1].length === 0 ? 1 : 0) -1;
const adjustedMaxDepth = this.maxCrawledDepth + baseURLDepth;
const crawler = new WebCrawler({
initialUrl: this.urls[0],
includes: this.includes,