Nick: fixed prettier
This commit is contained in:
@@ -5,7 +5,7 @@ import {
|
||||
RequestWithAuth,
|
||||
ScrapeRequest,
|
||||
scrapeRequestSchema,
|
||||
ScrapeResponse,
|
||||
ScrapeResponse
|
||||
} from "./types";
|
||||
import { billTeam } from "../../services/billing/credit_billing";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
@@ -30,7 +30,7 @@ export async function scrapeController(
|
||||
const jobPriority = await getJobPriority({
|
||||
plan: req.auth.plan as PlanType,
|
||||
team_id: req.auth.team_id,
|
||||
basePriority: 10,
|
||||
basePriority: 10
|
||||
});
|
||||
|
||||
await addScrapeJob(
|
||||
@@ -42,29 +42,37 @@ export async function scrapeController(
|
||||
internalOptions: {},
|
||||
plan: req.auth.plan!,
|
||||
origin: req.body.origin,
|
||||
is_scrape: true,
|
||||
is_scrape: true
|
||||
},
|
||||
{},
|
||||
jobId,
|
||||
jobPriority
|
||||
);
|
||||
|
||||
const totalWait = (req.body.waitFor ?? 0) + (req.body.actions ?? []).reduce((a,x) => (x.type === "wait" ? x.milliseconds ?? 0 : 0) + a, 0);
|
||||
const totalWait =
|
||||
(req.body.waitFor ?? 0) +
|
||||
(req.body.actions ?? []).reduce(
|
||||
(a, x) => (x.type === "wait" ? (x.milliseconds ?? 0) : 0) + a,
|
||||
0
|
||||
);
|
||||
|
||||
let doc: Document;
|
||||
try {
|
||||
doc = await waitForJob<Document>(jobId, timeout + totalWait); // TODO: better types for this
|
||||
} catch (e) {
|
||||
logger.error(`Error in scrapeController: ${e}`);
|
||||
if (e instanceof Error && (e.message.startsWith("Job wait") || e.message === "timeout")) {
|
||||
if (
|
||||
e instanceof Error &&
|
||||
(e.message.startsWith("Job wait") || e.message === "timeout")
|
||||
) {
|
||||
return res.status(408).json({
|
||||
success: false,
|
||||
error: "Request timed out",
|
||||
error: "Request timed out"
|
||||
});
|
||||
} else {
|
||||
return res.status(500).json({
|
||||
success: false,
|
||||
error: `(Internal server error) - ${(e && e.message) ? e.message : e}`,
|
||||
error: `(Internal server error) - ${e && e.message ? e.message : e}`
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -75,8 +83,8 @@ export async function scrapeController(
|
||||
const timeTakenInSeconds = (endTime - startTime) / 1000;
|
||||
const numTokens =
|
||||
doc && doc.extract
|
||||
// ? numTokensFromString(doc.markdown, "gpt-3.5-turbo")
|
||||
? 0 // TODO: fix
|
||||
? // ? numTokensFromString(doc.markdown, "gpt-3.5-turbo")
|
||||
0 // TODO: fix
|
||||
: 0;
|
||||
|
||||
let creditsToBeBilled = 1; // Assuming 1 credit per document
|
||||
@@ -84,14 +92,18 @@ export async function scrapeController(
|
||||
// Don't bill if we're early returning
|
||||
return;
|
||||
}
|
||||
if(req.body.extract && req.body.formats.includes("extract")) {
|
||||
if (req.body.extract && req.body.formats.includes("extract")) {
|
||||
creditsToBeBilled = 5;
|
||||
}
|
||||
|
||||
billTeam(req.auth.team_id, req.acuc?.sub_id, creditsToBeBilled).catch(error => {
|
||||
logger.error(`Failed to bill team ${req.auth.team_id} for ${creditsToBeBilled} credits: ${error}`);
|
||||
// Optionally, you could notify an admin or add to a retry queue here
|
||||
});
|
||||
billTeam(req.auth.team_id, req.acuc?.sub_id, creditsToBeBilled).catch(
|
||||
(error) => {
|
||||
logger.error(
|
||||
`Failed to bill team ${req.auth.team_id} for ${creditsToBeBilled} credits: ${error}`
|
||||
);
|
||||
// Optionally, you could notify an admin or add to a retry queue here
|
||||
}
|
||||
);
|
||||
|
||||
if (!req.body.formats.includes("rawHtml")) {
|
||||
if (doc && doc.rawHtml) {
|
||||
@@ -111,12 +123,12 @@ export async function scrapeController(
|
||||
url: req.body.url,
|
||||
scrapeOptions: req.body,
|
||||
origin: origin,
|
||||
num_tokens: numTokens,
|
||||
num_tokens: numTokens
|
||||
});
|
||||
|
||||
return res.status(200).json({
|
||||
success: true,
|
||||
data: doc,
|
||||
scrape_id: origin?.includes("website") ? jobId : undefined,
|
||||
scrape_id: origin?.includes("website") ? jobId : undefined
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user