Nick: fixed prettier

This commit is contained in:
Nicolas
2024-12-11 19:46:11 -03:00
parent e5fe9e1534
commit 00335e2ba9
134 changed files with 9565 additions and 7108 deletions
+28 -16
View File
@@ -5,7 +5,7 @@ import {
RequestWithAuth,
ScrapeRequest,
scrapeRequestSchema,
ScrapeResponse,
ScrapeResponse
} from "./types";
import { billTeam } from "../../services/billing/credit_billing";
import { v4 as uuidv4 } from "uuid";
@@ -30,7 +30,7 @@ export async function scrapeController(
const jobPriority = await getJobPriority({
plan: req.auth.plan as PlanType,
team_id: req.auth.team_id,
basePriority: 10,
basePriority: 10
});
await addScrapeJob(
@@ -42,29 +42,37 @@ export async function scrapeController(
internalOptions: {},
plan: req.auth.plan!,
origin: req.body.origin,
is_scrape: true,
is_scrape: true
},
{},
jobId,
jobPriority
);
const totalWait = (req.body.waitFor ?? 0) + (req.body.actions ?? []).reduce((a,x) => (x.type === "wait" ? x.milliseconds ?? 0 : 0) + a, 0);
const totalWait =
(req.body.waitFor ?? 0) +
(req.body.actions ?? []).reduce(
(a, x) => (x.type === "wait" ? (x.milliseconds ?? 0) : 0) + a,
0
);
let doc: Document;
try {
doc = await waitForJob<Document>(jobId, timeout + totalWait); // TODO: better types for this
} catch (e) {
logger.error(`Error in scrapeController: ${e}`);
if (e instanceof Error && (e.message.startsWith("Job wait") || e.message === "timeout")) {
if (
e instanceof Error &&
(e.message.startsWith("Job wait") || e.message === "timeout")
) {
return res.status(408).json({
success: false,
error: "Request timed out",
error: "Request timed out"
});
} else {
return res.status(500).json({
success: false,
error: `(Internal server error) - ${(e && e.message) ? e.message : e}`,
error: `(Internal server error) - ${e && e.message ? e.message : e}`
});
}
}
@@ -75,8 +83,8 @@ export async function scrapeController(
const timeTakenInSeconds = (endTime - startTime) / 1000;
const numTokens =
doc && doc.extract
// ? numTokensFromString(doc.markdown, "gpt-3.5-turbo")
? 0 // TODO: fix
? // ? numTokensFromString(doc.markdown, "gpt-3.5-turbo")
0 // TODO: fix
: 0;
let creditsToBeBilled = 1; // Assuming 1 credit per document
@@ -84,14 +92,18 @@ export async function scrapeController(
// Don't bill if we're early returning
return;
}
if(req.body.extract && req.body.formats.includes("extract")) {
if (req.body.extract && req.body.formats.includes("extract")) {
creditsToBeBilled = 5;
}
billTeam(req.auth.team_id, req.acuc?.sub_id, creditsToBeBilled).catch(error => {
logger.error(`Failed to bill team ${req.auth.team_id} for ${creditsToBeBilled} credits: ${error}`);
// Optionally, you could notify an admin or add to a retry queue here
});
billTeam(req.auth.team_id, req.acuc?.sub_id, creditsToBeBilled).catch(
(error) => {
logger.error(
`Failed to bill team ${req.auth.team_id} for ${creditsToBeBilled} credits: ${error}`
);
// Optionally, you could notify an admin or add to a retry queue here
}
);
if (!req.body.formats.includes("rawHtml")) {
if (doc && doc.rawHtml) {
@@ -111,12 +123,12 @@ export async function scrapeController(
url: req.body.url,
scrapeOptions: req.body,
origin: origin,
num_tokens: numTokens,
num_tokens: numTokens
});
return res.status(200).json({
success: true,
data: doc,
scrape_id: origin?.includes("website") ? jobId : undefined,
scrape_id: origin?.includes("website") ? jobId : undefined
});
}