Merge pull request #754 from mendableai/nsc/no-cluster-all

Rm cluster mode + rm fly deployments

.github/workflows/fly-direct.yml

@@ -1,46 +0,0 @@
-name: Fly Deploy Direct
-on:
-  schedule:
-    - cron: '0 * * * *'
-
-env:
-  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
-  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
-  HOST: ${{ secrets.HOST }}
-  LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
-  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
-  POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
-  POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
-  NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
-  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-  PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
-  PORT: ${{ secrets.PORT }}
-  REDIS_URL: ${{ secrets.REDIS_URL }}
-  SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }}
-  SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
-  SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
-  SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
-  TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
-  PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
-  PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-  NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-  CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
-  SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-  USE_DB_AUTHENTICATION: ${{ secrets.USE_DB_AUTHENTICATION }}
-  ENV: ${{ secrets.ENV }}
-
-jobs:
-  deploy:
-    name: Deploy app
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    steps:
-      - uses: actions/checkout@v3
-      - uses: superfly/flyctl-actions/setup-flyctl@master
-      - run: flyctl deploy --remote-only -a firecrawl-scraper-js --build-secret SENTRY_AUTH_TOKEN=$SENTRY_AUTH_TOKEN
-        working-directory: ./apps/api
-        env:
-          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
-          BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}

.github/workflows/fly.yml

@@ -1,81 +0,0 @@
-name: Fly Deploy
-on:
-  push:
-    branches:
-      - main
-
-env:
-  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
-  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
-  HOST: ${{ secrets.HOST }}
-  LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
-  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
-  POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
-  POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
-  NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
-  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-  PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
-  PORT: ${{ secrets.PORT }}
-  REDIS_URL: ${{ secrets.REDIS_URL }}
-  SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }}
-  SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
-  SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
-  SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
-  TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
-  PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
-  PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-  NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-  CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
-  SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-  USE_DB_AUTHENTICATION: ${{ secrets.USE_DB_AUTHENTICATION }}
-  ENV: ${{ secrets.ENV }}
-
-jobs:
-  pre-deploy:
-    name: Pre-deploy checks
-    runs-on: ubuntu-latest
-    services:
-      redis:
-        image: redis
-        ports:
-          - 6379:6379
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "20"
-      - name: Install pnpm
-        run: npm install -g pnpm
-      - name: Install dependencies
-        run: pnpm install
-        working-directory: ./apps/api
-      - name: Start the application
-        run: npm start &
-        working-directory: ./apps/api
-        id: start_app
-      - name: Start workers
-        run: npm run workers &
-        working-directory: ./apps/api
-        id: start_workers
-      - name: Run E2E tests
-        run: |
-          npm run test:prod
-        working-directory: ./apps/api
-
-  deploy:
-    name: Deploy app
-    needs: pre-deploy
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: superfly/flyctl-actions/setup-flyctl@master
-      - run: flyctl deploy --remote-only -a firecrawl-scraper-js --build-secret SENTRY_AUTH_TOKEN=$SENTRY_AUTH_TOKEN
-        working-directory: ./apps/api
-        env:
-          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
-          BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-
-  

apps/api/Dockerfile

@@ -28,9 +28,6 @@ RUN cd /app/src/lib/go-html-to-md && \
     chmod +x html-to-markdown.so
 
 FROM base
-RUN apt-get update -qq && \
-    apt-get install --no-install-recommends -y chromium chromium-sandbox && \
-    rm -rf /var/lib/apt/lists /var/cache/apt/archives
 COPY --from=prod-deps /app/node_modules /app/node_modules
 COPY --from=build /app /app
 COPY --from=go-base /app/src/lib/go-html-to-md/html-to-markdown.so /app/dist/src/lib/go-html-to-md/html-to-markdown.so

apps/api/src/index.ts

@@ -7,7 +7,6 @@ import cors from "cors";
 import { getScrapeQueue } from "./services/queue-service";
 import { v0Router } from "./routes/v0";
 import { initSDK } from "@hyperdx/node-opentelemetry";
-import cluster from "cluster";
 import os from "os";
 import { Logger } from "./lib/logger";
 import { adminRouter } from "./routes/admin";
@@ -37,22 +36,6 @@ const cacheable = new CacheableLookup({
 cacheable.install(http.globalAgent);
 cacheable.install(https.globalAgent)
 
-if (cluster.isMaster) {
-  Logger.info(`Master ${process.pid} is running`);
-
-  // Fork workers.
-  for (let i = 0; i < numCPUs; i++) {
-    cluster.fork();
-  }
-
-  cluster.on("exit", (worker, code, signal) => {
-    if (code !== null) {
-      Logger.info(`Worker ${worker.process.pid} exited`);
-      Logger.info("Starting a new worker");
-      cluster.fork();
-    }
-  });
-} else {
 const ws = expressWs(express());
 const app = ws.app;
 
@@ -226,9 +209,6 @@ if (cluster.isMaster) {
 });
 
 Logger.info(`Worker ${process.pid} started`);
-}
-
-
 
 // const sq = getScrapeQueue();