feat(crawl): URL deduplication

2024-11-08 16:22:06 +01:00
parent 25e94ffd28
commit 6ecf24b85e
4 changed files with 50 additions and 7 deletions
@@ -86,6 +86,9 @@ export interface CrawlScrapeOptions {
    country?: string;
    languages?: string[];
  };
+  skipTlsVerification?: boolean;
+  removeBase64Images?: boolean;
+  deduplicateSimilarURLs?: boolean;
 }

 export type Action = {