minor spacing and comment stuff

2024-07-01 16:05:34 -03:00
parent a5fb45988c
commit 4d6e25619b
1 changed files with 5 additions and 7 deletions
@@ -286,8 +286,6 @@ export class WebCrawler {
          if (this.isInternalLink(fullUrl)) { // INTERNAL LINKS
            if (this.isInternalLink(fullUrl) &&
              this.noSections(fullUrl) &&
-              // The idea here to comment this out is to allow wider website coverage as we filter this anyway afterwards
-              // this.matchesIncludes(path) &&
              !this.matchesExcludes(path) &&
              this.isRobotsAllowed(fullUrl)
            ) {
@@ -295,7 +293,7 @@ export class WebCrawler {
            }
          } else { // EXTERNAL LINKS
            if (
-              this.isInternalLink(url) && //its avoid to add links from external pages on the queue
+              this.isInternalLink(url) &&
              this.allowExternalContentLinks &&
              !this.isSocialMediaOrEmail(fullUrl) &&
              !this.matchesExcludes(fullUrl, true) &&