Merge branch 'main' into feat/issue-205

2024-06-14 11:25:20 -03:00
parent e37d151404 5fd228f9ec
commit 3e2e76311c
18 changed files with 400 additions and 202 deletions
@@ -479,7 +479,8 @@ export class WebScraperDataProvider {
      onlyMainContent: false,
      includeHtml: false,
      replaceAllPathsWithAbsolutePaths: false,
-      parsePDF: true
+      parsePDF: true,
+      removeTags: []
    };
    this.extractorOptions = options.extractorOptions ?? {mode: "markdown"}
    this.replaceAllPathsWithAbsolutePaths = options.crawlerOptions?.replaceAllPathsWithAbsolutePaths ?? options.pageOptions?.replaceAllPathsWithAbsolutePaths ?? false;