v1 restructure

apps/api/src/controllers/v1/crawl.ts

@@ -1,20 +1,16 @@
 import { Request, Response } from "express";
-import { WebScraperDataProvider } from "../../../src/scraper/WebScraper";
-import { billTeam } from "../../../src/services/billing/credit_billing";
 import { checkTeamCredits } from "../../../src/services/billing/credit_billing";
 import { authenticateUser } from "./auth";
 import { RateLimiterMode } from "../../../src/types";
-import { addWebScraperJob } from "../../../src/services/queue-jobs";
 import { isUrlBlocked } from "../../../src/scraper/WebScraper/utils/blocklist";
-import { logCrawl } from "../../../src/services/logging/crawl_log";
 import { validateIdempotencyKey } from "../../../src/services/idempotency/validate";
 import { createIdempotencyKey } from "../../../src/services/idempotency/create";
-import { defaultCrawlPageOptions, defaultCrawlerOptions, defaultOrigin } from "../../../src/lib/default-values";
 import { v4 as uuidv4 } from "uuid";
 import { Logger } from "../../../src/lib/logger";
 import { checkAndUpdateURL } from "../../../src/lib/validateUrl";
+import { CrawlRequest, CrawlResponse } from "./types";
 
-export async function crawlController(req: Request, res: Response) {
+export async function crawlController(req: Request<{}, {}, CrawlRequest>, res: Response<CrawlResponse>) {
   // expected req.body
 
   // req.body = {
@@ -39,52 +35,57 @@ export async function crawlController(req: Request, res: Response) {
       RateLimiterMode.Crawl
     );
     if (!success) {
-      return res.status(status).json({ error });
+      return res.status(status).json({ success: false, error });
     }
 
     if (req.headers["x-idempotency-key"]) {
       const isIdempotencyValid = await validateIdempotencyKey(req);
       if (!isIdempotencyValid) {
-        return res.status(409).json({ error: "Idempotency key already used" });
+        return res.status(409).json({ success: false, error: "Idempotency key already used" });
       }
       try {
         createIdempotencyKey(req);
       } catch (error) {
         Logger.error(error);
-        return res.status(500).json({ error: error.message });
+        return res.status(500).json({ success: false, error: error.message });
       }
     }
 
     const { success: creditsCheckSuccess, message: creditsCheckMessage } =
       await checkTeamCredits(team_id, 1);
     if (!creditsCheckSuccess) {
-      return res.status(402).json({ error: "Insufficient credits" });
+      return res.status(402).json({ success: false, error: "Insufficient credits" });
     }
 
     let url = req.body.url;
     if (!url) {
-      return res.status(400).json({ error: "Url is required" });
+      return res.status(400).json({ success: false, error: "Url is required" });
     }
 
     if (isUrlBlocked(url)) {
       return res
         .status(403)
         .json({
+          success: false,
           error:
             "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",
         });
     }
 
     try {
-      url = checkAndUpdateURL(url);
+      url = checkAndUpdateURL(url).url;
     } catch (error) {
-      return res.status(400).json({ error: 'Invalid Url' });
+      return res.status(400).json({ success: false, error: 'Invalid Url' });
     }
 
     // TODO: add job to queue
 
     const id = uuidv4();
-    return res.status(200).json({ jobId: id, url: `${req.protocol}://${req.get('host')}/v1/crawl/${id}` });
+    return res.status(200).json({
+      success: true,
+      id,
+      url: `${req.protocol}://${req.get('host')}/v1/crawl/${id}`,
+    });
 
     // const mode = req.body.mode ?? "crawl";
 
@@ -134,6 +135,6 @@ export async function crawlController(req: Request, res: Response) {
     // res.json({ jobId: job.id });
   } catch (error) {
     Logger.error(error);
-    return res.status(500).json({ error: error.message });
+    return res.status(500).json({ success: false, error: error.message });
   }
 }