Nick: fixes

This commit is contained in:
Nicolas
2024-12-11 19:48:22 -03:00
parent 00335e2ba9
commit 52f2e733e2
6 changed files with 43 additions and 44 deletions
+2 -1
View File
@@ -6,7 +6,7 @@
"scripts": { "scripts": {
"start": "nodemon --exec ts-node src/index.ts", "start": "nodemon --exec ts-node src/index.ts",
"start:production": "tsc && node dist/src/index.js", "start:production": "tsc && node dist/src/index.js",
"format": "npx prettier --write \"src/**/*.(js|ts)\"", "format": "prettier --write \"src/**/*.(js|ts)\"",
"flyio": "node dist/src/index.js", "flyio": "node dist/src/index.js",
"start:dev": "nodemon --exec ts-node src/index.ts", "start:dev": "nodemon --exec ts-node src/index.ts",
"build": "tsc && pnpm sentry:sourcemaps", "build": "tsc && pnpm sentry:sourcemaps",
@@ -102,6 +102,7 @@
"pdf-parse": "^1.1.1", "pdf-parse": "^1.1.1",
"pos": "^0.4.2", "pos": "^0.4.2",
"posthog-node": "^4.0.1", "posthog-node": "^4.0.1",
"prettier": "^3.4.2",
"promptable": "^0.0.10", "promptable": "^0.0.10",
"puppeteer": "^22.12.1", "puppeteer": "^22.12.1",
"rate-limiter-flexible": "2.4.2", "rate-limiter-flexible": "2.4.2",
+14 -4
View File
@@ -164,6 +164,9 @@ importers:
posthog-node: posthog-node:
specifier: ^4.0.1 specifier: ^4.0.1
version: 4.0.1 version: 4.0.1
prettier:
specifier: ^3.4.2
version: 3.4.2
promptable: promptable:
specifier: ^0.0.10 specifier: ^0.0.10
version: 0.0.10 version: 0.0.10
@@ -3756,6 +3759,11 @@ packages:
resolution: {integrity: sha512-rtqm2h22QxLGBrW2bLYzbRhliIrqgZ0k+gF0LkQ1SNdeD06YE5eilV0MxZppFSxC8TfH0+B0cWCuebEnreIDgQ==} resolution: {integrity: sha512-rtqm2h22QxLGBrW2bLYzbRhliIrqgZ0k+gF0LkQ1SNdeD06YE5eilV0MxZppFSxC8TfH0+B0cWCuebEnreIDgQ==}
engines: {node: '>=15.0.0'} engines: {node: '>=15.0.0'}
prettier@3.4.2:
resolution: {integrity: sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ==}
engines: {node: '>=14'}
hasBin: true
pretty-format@29.7.0: pretty-format@29.7.0:
resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==} resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
@@ -4321,8 +4329,8 @@ packages:
engines: {node: '>=14.17'} engines: {node: '>=14.17'}
hasBin: true hasBin: true
typescript@5.6.3: typescript@5.7.2:
resolution: {integrity: sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==} resolution: {integrity: sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==}
engines: {node: '>=14.17'} engines: {node: '>=14.17'}
hasBin: true hasBin: true
@@ -8978,6 +8986,8 @@ snapshots:
transitivePeerDependencies: transitivePeerDependencies:
- debug - debug
prettier@3.4.2: {}
pretty-format@29.7.0: pretty-format@29.7.0:
dependencies: dependencies:
'@jest/schemas': 29.6.3 '@jest/schemas': 29.6.3
@@ -9000,7 +9010,7 @@ snapshots:
csv-parse: 5.5.6 csv-parse: 5.5.6
gpt3-tokenizer: 1.1.5 gpt3-tokenizer: 1.1.5
openai: 3.3.0 openai: 3.3.0
typescript: 5.6.3 typescript: 5.7.2
uuid: 9.0.1 uuid: 9.0.1
zod: 3.23.8 zod: 3.23.8
transitivePeerDependencies: transitivePeerDependencies:
@@ -9584,7 +9594,7 @@ snapshots:
typescript@5.4.5: {} typescript@5.4.5: {}
typescript@5.6.3: {} typescript@5.7.2: {}
typesense@1.8.2(@babel/runtime@7.24.6): typesense@1.8.2(@babel/runtime@7.24.6):
dependencies: dependencies:
+2 -6
View File
@@ -86,9 +86,7 @@ export async function crawlController(req: Request, res: Response) {
} = await checkTeamCredits(chunk, team_id, limitCheck); } = await checkTeamCredits(chunk, team_id, limitCheck);
if (!creditsCheckSuccess) { if (!creditsCheckSuccess) {
return res return res.status(402).json({
.status(402)
.json({
error: error:
"Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. If not, upgrade your plan at https://firecrawl.dev/pricing or contact us at help@firecrawl.com" "Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. If not, upgrade your plan at https://firecrawl.dev/pricing or contact us at help@firecrawl.com"
}); });
@@ -259,9 +257,7 @@ export async function crawlController(req: Request, res: Response) {
} catch (error) { } catch (error) {
Sentry.captureException(error); Sentry.captureException(error);
logger.error(error); logger.error(error);
return res return res.status(500).json({
.status(500)
.json({
error: error instanceof ZodError ? "Invalid URL" : error.message error: error instanceof ZodError ? "Invalid URL" : error.message
}); });
} }
+1 -3
View File
@@ -211,9 +211,7 @@ export async function scrapeController(req: Request, res: Response) {
await checkTeamCredits(chunk, team_id, 1); await checkTeamCredits(chunk, team_id, 1);
if (!creditsCheckSuccess) { if (!creditsCheckSuccess) {
earlyReturn = true; earlyReturn = true;
return res return res.status(402).json({
.status(402)
.json({
error: error:
"Insufficient credits. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing" "Insufficient credits. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing"
}); });
+1 -3
View File
@@ -235,9 +235,7 @@ app.use(
" -- " + " -- " +
verbose verbose
); );
res res.status(500).json({
.status(500)
.json({
success: false, success: false,
error: error:
"An unexpected error occurred. Please contact help@firecrawl.com for help. Your exception ID is " + "An unexpected error occurred. Please contact help@firecrawl.com for help. Your exception ID is " +
+2 -6
View File
@@ -54,9 +54,7 @@ function checkCreditsMiddleware(
`Insufficient credits: ${JSON.stringify({ team_id: req.auth.team_id, minimum, remainingCredits })}` `Insufficient credits: ${JSON.stringify({ team_id: req.auth.team_id, minimum, remainingCredits })}`
); );
if (!res.headersSent) { if (!res.headersSent) {
return res return res.status(402).json({
.status(402)
.json({
success: false, success: false,
error: error:
"Insufficient credits to perform this request. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing or try changing the request limit to a lower value." "Insufficient credits to perform this request. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing or try changing the request limit to a lower value."
@@ -122,9 +120,7 @@ function idempotencyMiddleware(
function blocklistMiddleware(req: Request, res: Response, next: NextFunction) { function blocklistMiddleware(req: Request, res: Response, next: NextFunction) {
if (typeof req.body.url === "string" && isUrlBlocked(req.body.url)) { if (typeof req.body.url === "string" && isUrlBlocked(req.body.url)) {
if (!res.headersSent) { if (!res.headersSent) {
return res return res.status(403).json({
.status(403)
.json({
success: false, success: false,
error: error:
"URL is blocked intentionally. Firecrawl currently does not support social media scraping due to policy restrictions." "URL is blocked intentionally. Firecrawl currently does not support social media scraping due to policy restrictions."