Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

minor spacing and comment stuff

Browse Source
This commit is contained in:
rafaelsideguide
2024-07-01 16:05:34 -03:00
parent a5fb45988c
commit 4d6e25619b
1 changed files with 5 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/crawler.ts
+1 -3
View File
@@ -286,8 +286,6 @@ export class WebCrawler {
if (this.isInternalLink(fullUrl)) { // INTERNAL LINKS if (this.isInternalLink(fullUrl)) { // INTERNAL LINKS
if (this.isInternalLink(fullUrl) && if (this.isInternalLink(fullUrl) &&
this.noSections(fullUrl) && this.noSections(fullUrl) &&
// The idea here to comment this out is to allow wider website coverage as we filter this anyway afterwards
// this.matchesIncludes(path) &&
!this.matchesExcludes(path) && !this.matchesExcludes(path) &&
this.isRobotsAllowed(fullUrl) this.isRobotsAllowed(fullUrl)
) { ) {
@@ -295,7 +293,7 @@ export class WebCrawler {
} }
} else { // EXTERNAL LINKS } else { // EXTERNAL LINKS
if ( if (
this.isInternalLink(url) && //its avoid to add links from external pages on the queue this.isInternalLink(url) &&
this.allowExternalContentLinks && this.allowExternalContentLinks &&
!this.isSocialMediaOrEmail(fullUrl) && !this.isSocialMediaOrEmail(fullUrl) &&
!this.matchesExcludes(fullUrl, true) && !this.matchesExcludes(fullUrl, true) &&