Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update extract.ts

Browse Source
This commit is contained in:
Nicolas
2024-11-20 16:40:55 -08:00
parent d196b9d93d
commit 945183ffbd
1 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/controllers/v1/extract.ts
+4 -5
View File
@@ -87,7 +87,6 @@ export async function extractController(
let mappedLinksRerank = mappedLinks.map(x => `url: ${x.url}, title: ${x.title}, description: ${x.description}`); let mappedLinksRerank = mappedLinks.map(x => `url: ${x.url}, title: ${x.title}, description: ${x.description}`);
// Filter by path prefix if present // Filter by path prefix if present
// console.log("pathPrefix", pathPrefix);
// wrong // wrong
// if (pathPrefix) { // if (pathPrefix) {
// mappedLinks = mappedLinks.filter(x => x.url && x.url.includes(`/${pathPrefix}/`)); // mappedLinks = mappedLinks.filter(x => x.url && x.url.includes(`/${pathPrefix}/`));
@@ -100,8 +99,8 @@ export async function extractController(
.map(x => mappedLinks.find(link => link.url === x.link)) .map(x => mappedLinks.find(link => link.url === x.link))
.filter((x): x is MapDocument => x !== undefined && x.url !== undefined && !isUrlBlocked(x.url)) .filter((x): x is MapDocument => x !== undefined && x.url !== undefined && !isUrlBlocked(x.url))
.slice(0, MAX_RANKING_LIMIT); .slice(0, MAX_RANKING_LIMIT);
console.log("linksAndScores", linksAndScores); // console.log("linksAndScores", linksAndScores);
console.log("linksAndScores", linksAndScores.length); // console.log("linksAndScores", linksAndScores.length);
} }
return mappedLinks.map(x => x.url) as string[]; return mappedLinks.map(x => x.url) as string[];
@@ -119,7 +118,7 @@ export async function extractController(
const processedUrls = await Promise.all(urlPromises); const processedUrls = await Promise.all(urlPromises);
links.push(...processedUrls.flat()); links.push(...processedUrls.flat());
console.log("links", links.length); // console.log("links", links.length);
// Scrape all links in parallel // Scrape all links in parallel
const scrapePromises = links.map(async (url) => { const scrapePromises = links.map(async (url) => {
const origin = req.body.origin || "api"; const origin = req.body.origin || "api";
@@ -208,7 +207,7 @@ export async function extractController(
}); });
console.log("completions.extract", completions.extract); // console.log("completions.extract", completions.extract);
let data: any; let data: any;
try { try {