diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index a7be4fe0..201b2ce9 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -12,8 +12,11 @@ RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-l FROM base AS build RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile +RUN apt-get update -qq && apt-get install -y ca-certificates && update-ca-certificates RUN pnpm install -RUN pnpm run build +RUN --mount=type=secret,id=SENTRY_AUTH_TOKEN \ + SENTRY_AUTH_TOKEN="$(cat /run/secrets/SENTRY_AUTH_TOKEN)" \ + bash -c "if [ -z $SENTRY_AUTH_TOKEN ]; then pnpm run build:nosentry; else pnpm run build; fi" # Install packages needed for deployment diff --git a/apps/api/package.json b/apps/api/package.json index 55cefe96..c9058943 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -5,11 +5,12 @@ "main": "index.js", "scripts": { "start": "nodemon --exec ts-node src/index.ts", - "start:production": "tsc && pnpm sentry:sourcemaps && node dist/src/index.js", + "start:production": "tsc && node dist/src/index.js", "format": "prettier --write \"src/**/*.(js|ts)\"", "flyio": "node dist/src/index.js", "start:dev": "nodemon --exec ts-node src/index.ts", "build": "tsc && pnpm sentry:sourcemaps", + "build:nosentry": "tsc", "test": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_noAuth/*'", "test:local-no-auth": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_withAuth/*'", "test:full": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)'", @@ -19,7 +20,7 @@ "mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest", "mongo-docker-console": "docker exec -it mongodb mongosh", "run-example": "npx ts-node src/example.ts", - "deploy:fly": "flyctl deploy", + "deploy:fly": "flyctl deploy --build-secret SENTRY_AUTH_TOKEN=$(dotenv -p SENTRY_AUTH_TOKEN)", "deploy:fly:staging": "fly deploy -c fly.staging.toml", "sentry:sourcemaps": "sentry-cli sourcemaps inject --org caleb-peffer --project firecrawl-scraper-js ./dist && sentry-cli sourcemaps upload --org caleb-peffer --project firecrawl-scraper-js ./dist" }, @@ -73,6 +74,7 @@ "cron-parser": "^4.9.0", "date-fns": "^3.6.0", "dotenv": "^16.3.1", + "dotenv-cli": "^7.4.2", "express-rate-limit": "^7.3.1", "form-data": "^4.0.0", "glob": "^10.4.2", diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index 3f59352c..efbe9d80 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -86,6 +86,9 @@ importers: dotenv: specifier: ^16.3.1 version: 16.4.5 + dotenv-cli: + specifier: ^7.4.2 + version: 7.4.2 express-rate-limit: specifier: ^7.3.1 version: 7.3.1(express@4.19.2) @@ -2367,6 +2370,14 @@ packages: domutils@3.1.0: resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} + dotenv-cli@7.4.2: + resolution: {integrity: sha512-SbUj8l61zIbzyhIbg0FwPJq6+wjbzdn9oEtozQpZ6kW2ihCcapKVZj49oCT3oPM+mgQm+itgvUQcG5szxVrZTA==} + hasBin: true + + dotenv-expand@10.0.0: + resolution: {integrity: sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==} + engines: {node: '>=12'} + dotenv@16.4.5: resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} engines: {node: '>=12'} @@ -7396,6 +7407,15 @@ snapshots: domelementtype: 2.3.0 domhandler: 5.0.3 + dotenv-cli@7.4.2: + dependencies: + cross-spawn: 7.0.3 + dotenv: 16.4.5 + dotenv-expand: 10.0.0 + minimist: 1.2.8 + + dotenv-expand@10.0.0: {} + dotenv@16.4.5: {} duck@0.1.12: diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index 51f995c6..7d4beef1 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -1,5 +1,5 @@ import "dotenv/config"; -import "./services/sentry" +import "./sentry" import * as Sentry from "@sentry/node"; import { CustomError } from "../lib/custom-error"; import { @@ -105,8 +105,11 @@ const workerFun = async (queueName: string, processJobInternal: (token: string, const job = await worker.getNextJob(token); if (job) { Sentry.startSpan({ - name: "Job " + job.id, - parentSpan: null, + name: "Scrape job", + op: "bullmq.job", + attributes: { + job: job.id, + }, }, async () => { await processJobInternal(token, job); });