Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: remove webScraperQueue

Browse Source
This commit is contained in:
Gergo Moricz
2024-08-13 21:03:24 +02:00
parent 4a2c37dcf5
commit d7549d4dc5
12 changed files with 151 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/controllers/crawl.ts
+1 -3
View File
@@ -1,10 +1,8 @@
import { Request, Response } from "express";
import { WebScraperDataProvider } from "../../src/scraper/WebScraper";
import { billTeam } from "../../src/services/billing/credit_billing";
import { checkTeamCredits } from "../../src/services/billing/credit_billing";
import { authenticateUser } from "./auth";
import { RateLimiterMode } from "../../src/types";
import { addScrapeJob, addWebScraperJob } from "../../src/services/queue-jobs";
import { addScrapeJob } from "../../src/services/queue-jobs";
import { isUrlBlocked } from "../../src/scraper/WebScraper/utils/blocklist";
import { logCrawl } from "../../src/services/logging/crawl_log";
import { validateIdempotencyKey } from "../../src/services/idempotency/validate";