Files
firecrawl/apps/api/src/scraper/WebScraper/utils/blocklist.ts
T
2024-04-23 18:50:35 -03:00

20 lines
383 B
TypeScript

const socialMediaBlocklist = [
'facebook.com',
'twitter.com',
'instagram.com',
'linkedin.com',
'pinterest.com',
'snapchat.com',
'tiktok.com',
'reddit.com',
'tumblr.com',
'flickr.com',
'whatsapp.com',
'wechat.com',
'telegram.org',
];
export function isUrlBlocked(url: string): boolean {
return socialMediaBlocklist.some(domain => url.includes(domain));
}