Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update extract.ts

Browse Source
This commit is contained in:
Nicolas
2024-11-24 19:36:18 -08:00
parent 95bea6a391
commit b693c6c23b
1 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/controllers/v1/extract.ts
+8 -2
View File
@@ -93,14 +93,20 @@ export async function extractController(
// } // }
if (req.body.prompt) { if (req.body.prompt) {
// Get similarity scores between the search query and each link's context
const linksAndScores : { link: string, linkWithContext: string, score: number, originalIndex: number }[] = await performRanking(mappedLinksRerank, mappedLinks.map(l => l.url), mapUrl); const linksAndScores : { link: string, linkWithContext: string, score: number, originalIndex: number }[] = await performRanking(mappedLinksRerank, mappedLinks.map(l => l.url), mapUrl);
mappedLinks = linksAndScores mappedLinks = linksAndScores
// Only keep links that have a similarity score above the threshold
.filter(x => x.score > SCORE_THRESHOLD) .filter(x => x.score > SCORE_THRESHOLD)
// Map back to the original link objects
.map(x => mappedLinks.find(link => link.url === x.link)) .map(x => mappedLinks.find(link => link.url === x.link))
// Remove any undefined links, links without URLs, and blocked URLs
.filter((x): x is MapDocument => x !== undefined && x.url !== undefined && !isUrlBlocked(x.url)) .filter((x): x is MapDocument => x !== undefined && x.url !== undefined && !isUrlBlocked(x.url))
// Limit the number of results
.slice(0, MAX_RANKING_LIMIT); .slice(0, MAX_RANKING_LIMIT);
// console.log("linksAndScores", linksAndScores);
// console.log("linksAndScores", linksAndScores.length); // TODO: handle case where no links are returned
} }
return mappedLinks.map(x => x.url) as string[]; return mappedLinks.map(x => x.url) as string[];