Update blocklist.ts · 064ce482c2 - firecrawl - MKD Repo

Iyeoluwads/firecrawl

Update blocklist.ts

This commit is contained in:

Nicolas

2024-10-09 14:41:23 -03:00

parent 075b63b57b

commit 064ce482c2

1 changed files with 1 additions and 0 deletions

									
										apps/api/src/scraper/WebScraper/utils/blocklist.ts
									
		+1
		
												View File
												
					@@ -40,6 +40,7 @@ const allowedKeywords = [

					  'tos',

					  'tos',

					  '://ads.tiktok.com',

					  '://ads.tiktok.com',

					  '://tiktok.com/business',

					  '://tiktok.com/business',

					  '://developers.facebook.com'

					];

					];

					export function isUrlBlocked(url: string): boolean {

					export function isUrlBlocked(url: string): boolean {