Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Nick:

Browse Source
This commit is contained in:
Nicolas
2024-09-01 13:52:34 -03:00
parent 979697df1c
commit b68a50fea3
2 changed files with 41 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/services/queue-worker.ts
+6 -16
View File
@@ -221,12 +221,10 @@ async function processJob(job: Job, token: string) {
// No idea what this does and when it is called. // No idea what this does and when it is called.
if (job.data.mode === "crawl" && !job.data.v1) { if (job.data.mode === "crawl" && !job.data.v1) {
callWebhook(job.data.team_id, job.id as string, data, job.data.webhook, job.data.v1).catch((error) => { callWebhook(job.data.team_id, job.id as string, data, job.data.webhook, job.data.v1);
Logger.error(`Error calling webhook for job (1 - mode crawl - v0) ${job.id} - ${error}`);
});
} }
if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) { if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) {
await callWebhook(job.data.team_id, job.data.crawl_id, data, job.data.webhook, job.data.v1); callWebhook(job.data.team_id, job.data.crawl_id, data, job.data.webhook, job.data.v1);
} }
if (job.data.crawl_id) { if (job.data.crawl_id) {
@@ -351,15 +349,11 @@ async function processJob(job: Job, token: string) {
}; };
// v0 web hooks, call when done with all the data // v0 web hooks, call when done with all the data
if (!job.data.v1) { if (!job.data.v1) {
callWebhook(job.data.team_id, job.data.crawl_id, data, job.data.webhook, job.data.v1, "crawl.completed").catch((error) => { callWebhook(job.data.team_id, job.data.crawl_id, data, job.data.webhook, job.data.v1, "crawl.completed");
Logger.error(`Error calling webhook for job ${job.id} - ${error}`);
});
} }
// v1 web hooks, call when done with no data, but with event completed // v1 web hooks, call when done with no data, but with event completed
if (job.data.v1 && job.data.webhook) { if (job.data.v1 && job.data.webhook) {
callWebhook(job.data.team_id, job.data.crawl_id, [], job.data.webhook, job.data.v1, "crawl.completed").catch((error) => { callWebhook(job.data.team_id, job.data.crawl_id, [], job.data.webhook, job.data.v1, "crawl.completed");
Logger.error(`Error calling webhook for job ${job.id} - ${error}`);
});
} }
} }
} }
@@ -407,14 +401,10 @@ async function processJob(job: Job, token: string) {
}; };
if (!job.data.v1 && (job.data.mode === "crawl" || job.data.crawl_id)) { if (!job.data.v1 && (job.data.mode === "crawl" || job.data.crawl_id)) {
callWebhook(job.data.team_id, job.data.crawl_id ?? job.id as string, data, job.data.webhook, job.data.v1).catch((error) => { callWebhook(job.data.team_id, job.data.crawl_id ?? job.id as string, data, job.data.webhook, job.data.v1);
Logger.error(`Error calling webhook for job (catch - v0) ${job.id} - ${error}`);
});
} }
if(job.data.v1) { if(job.data.v1) {
callWebhook(job.data.team_id, job.id as string, [], job.data.webhook, job.data.v1, "crawl.failed").catch((error) => { callWebhook(job.data.team_id, job.id as string, [], job.data.webhook, job.data.v1, "crawl.failed");
Logger.error(`Error calling webhook for job (catch - v1) ${job.id} - ${error}`);
});
} }
if (job.data.crawl_id) { if (job.data.crawl_id) {
apps/api/src/services/webhook.ts
+21 -6
View File
@@ -43,7 +43,12 @@ export const callWebhook = async (
} }
let dataToSend = []; let dataToSend = [];
if (data && data.result && data.result.links && data.result.links.length !== 0) { if (
data &&
data.result &&
data.result.links &&
data.result.links.length !== 0
) {
for (let i = 0; i < data.result.links.length; i++) { for (let i = 0; i < data.result.links.length; i++) {
if (v1) { if (v1) {
dataToSend.push( dataToSend.push(
@@ -59,14 +64,23 @@ export const callWebhook = async (
} }
} }
axios.post( axios
.post(
webhookUrl, webhookUrl,
{ {
success: !v1 ? data.success : eventType === "crawl.page" ? data.success : true, success: !v1
? data.success
: eventType === "crawl.page"
? data.success
: true,
type: eventType, type: eventType,
[v1 ? 'id' : 'jobId']: id, [v1 ? "id" : "jobId"]: id,
data: dataToSend, data: dataToSend,
error: !v1 ? data?.error || undefined : eventType === "crawl.page" ? data?.error || undefined : undefined, error: !v1
? data?.error || undefined
: eventType === "crawl.page"
? data?.error || undefined
: undefined,
}, },
{ {
headers: { headers: {
@@ -74,7 +88,8 @@ export const callWebhook = async (
}, },
timeout: 10000, // 10 seconds timeout timeout: 10000, // 10 seconds timeout
} }
).catch((error) => { )
.catch((error) => {
Logger.error( Logger.error(
`Error sending webhook for team ID: ${teamId}, error: ${error.message}` `Error sending webhook for team ID: ${teamId}, error: ${error.message}`
); );