Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update search.ts

Browse Source
This commit is contained in:
Nicolas
2025-01-02 20:40:38 -03:00
parent b244afbc82
commit 7ce780ac81
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/controllers/v1/search.ts
+4 -2
View File
@@ -112,7 +112,6 @@ export async function searchController(
}); });
// Filter blocked URLs early to avoid unnecessary billing // Filter blocked URLs early to avoid unnecessary billing
searchResults = searchResults.filter((r) => !isUrlBlocked(r.url));
if (searchResults.length > limit) { if (searchResults.length > limit) {
searchResults = searchResults.slice(0, limit); searchResults = searchResults.slice(0, limit);
} }
@@ -152,7 +151,10 @@ export async function searchController(
}); });
} }
// Scrape each result, handling timeouts individually // Filter out blocked URLs before scraping
searchResults = searchResults.filter((r) => !isUrlBlocked(r.url));
// Scrape each non-blocked result, handling timeouts individually
const scrapePromises = searchResults.map((result) => const scrapePromises = searchResults.map((result) =>
scrapeSearchResult(result, { scrapeSearchResult(result, {
teamId: req.auth.team_id, teamId: req.auth.team_id,