Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: crawl to scrape conversion

Browse Source
This commit is contained in:
Gergo Moricz
2024-08-13 20:51:43 +02:00
parent e28c415cf4
commit 86e136beca
8 changed files with 313 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/controllers/crawl-status.ts
+28 -39
View File
@@ -1,10 +1,9 @@
import { Request, Response } from "express";
import { authenticateUser } from "./auth";
import { RateLimiterMode } from "../../src/types";
import { addWebScraperJob } from "../../src/services/queue-jobs";
import { getWebScraperQueue } from "../../src/services/queue-service";
import { supabaseGetJobById } from "../../src/lib/supabase-jobs";
import { getScrapeQueue } from "../../src/services/queue-service";
import { Logger } from "../../src/lib/logger";
import { getCrawl, getCrawlJobs } from "../../src/lib/crawl-redis";
export async function crawlStatusController(req: Request, res: Response) {
try {
@@ -16,51 +15,41 @@ export async function crawlStatusController(req: Request, res: Response) {
if (!success) {
return res.status(status).json({ error });
}
const job = await getWebScraperQueue().getJob(req.params.jobId);
if (!job) {
// const job = await getWebScraperQueue().getJob(req.params.jobId);
// if (!job) {
// return res.status(404).json({ error: "Job not found" });
// }
// const isCancelled = await (await getWebScraperQueue().client).exists("cancelled:" + req.params.jobId);
const sc = await getCrawl(req.params.jobId);
if (!sc) {
return res.status(404).json({ error: "Job not found" });
}
const isCancelled = await (await getWebScraperQueue().client).exists("cancelled:" + req.params.jobId);
const jobIDs = await getCrawlJobs(req.params.jobId);
let progress = job.progress;
if(typeof progress !== 'object') {
progress = {
current: 0,
current_url: '',
total: 0,
current_step: '',
partialDocs: []
}
}
const {
current = 0,
current_url = '',
total = 0,
current_step = '',
partialDocs = []
} = progress as { current: number, current_url: string, total: number, current_step: string, partialDocs: any[] };
// let data = job.returnvalue;
// if (process.env.USE_DB_AUTHENTICATION === "true") {
// const supabaseData = await supabaseGetJobById(req.params.jobId);
let data = job.returnvalue;
if (process.env.USE_DB_AUTHENTICATION === "true") {
const supabaseData = await supabaseGetJobById(req.params.jobId);
// if (supabaseData) {
// data = supabaseData.docs;
// }
// }
if (supabaseData) {
data = supabaseData.docs;
}
}
const jobs = await Promise.all(jobIDs.map(x => getScrapeQueue().getJob(x)));
const jobStatuses = await Promise.all(jobs.map(x => x.getState()));
const jobStatus = sc.cancelled ? "failed" : jobStatuses.every(x => x === "completed") ? "completed" : jobStatuses.some(x => x === "failed") ? "failed" : "active";
const jobStatus = await job.getState();
const data = jobs.map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue);
res.json({
status: isCancelled ? "failed" : jobStatus,
// progress: job.progress(),
current,
current_url,
current_step,
total,
data: data && !isCancelled ? data : null,
partial_data: jobStatus == 'completed' && !isCancelled ? [] : partialDocs,
status: jobStatus,
current: jobStatuses.filter(x => x === "completed" || x === "failed").length,
total: jobs.length,
data: jobStatus === "completed" ? data : null,
partial_data: jobStatus === "completed" ? [] : data.filter(x => x !== null),
});
} catch (error) {
Logger.error(error);