Nick: fixed credits issue

apps/api/src/controllers/v0/scrape.ts

@@ -63,6 +63,7 @@ export async function scrapeHelper(
       pageOptions,
       extractorOptions,
       origin: req.body.origin ?? defaultOrigin,
+      is_scrape: true,
     },
     {},
     jobId,
@@ -179,9 +180,7 @@ export async function scrapeController(req: Request, res: Response) {
         typeof extractorOptions.extractionSchema !== "object" ||
         extractorOptions.extractionSchema === null
       ) {
-        return res
+        return res.status(400).json({
-          .status(400)
-          .json({
           error:
             "extractorOptions.extractionSchema must be an object if llm-extraction mode is specified",
         });
@@ -202,9 +201,7 @@ export async function scrapeController(req: Request, res: Response) {
     } catch (error) {
       Logger.error(error);
       earlyReturn = true;
-      return res
+      return res.status(500).json({
-        .status(500)
-        .json({
         error:
           "Error checking team credits. Please contact hello@firecrawl.com for help.",
       });
@@ -231,8 +228,8 @@ export async function scrapeController(req: Request, res: Response) {
         : 0;
 
     if (result.success) {
-      let creditsToBeBilled = 0; // billing for doc done on queue end
+      let creditsToBeBilled = 1;
-      const creditsPerLLMExtract = 50;
+      const creditsPerLLMExtract = 49;
 
       if (extractorOptions.mode.includes("llm-extraction")) {
         // creditsToBeBilled = creditsToBeBilled + (creditsPerLLMExtract * filteredDocs.length);
@@ -245,6 +242,8 @@ export async function scrapeController(req: Request, res: Response) {
         // Don't bill if we're early returning
         return;
       }
+      if (creditsToBeBilled > 0) {
+        // billing for doc done on queue end, bill only for llm extraction
         const billingResult = await billTeam(team_id, creditsToBeBilled);
         if (!billingResult.success) {
           return res.status(402).json({
@@ -254,6 +253,7 @@ export async function scrapeController(req: Request, res: Response) {
           });
         }
       }
+    }
 
     logJob({
       job_id: jobId,
@@ -276,9 +276,7 @@ export async function scrapeController(req: Request, res: Response) {
   } catch (error) {
     Sentry.captureException(error);
     Logger.error(error);
-    return res
+    return res.status(500).json({
-      .status(500)
-      .json({
       error:
         typeof error === "string"
           ? error

apps/api/src/controllers/v1/scrape.ts

@@ -29,6 +29,7 @@ export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse,
     pageOptions,
     extractorOptions: {},
     origin: req.body.origin,
+    is_scrape: true,
   }, {}, jobId, jobPriority);
 
   let doc: any | undefined;

apps/api/src/main/runWebScraper.ts

@@ -57,6 +57,7 @@ export async function startWebScraperPipeline({
     team_id: job.data.team_id,
     bull_job_id: job.id.toString(),
     priority: job.opts.priority,
+    is_scrape: job.data.is_scrape ?? false,
   })) as { success: boolean; message: string; docs: Document[] };
 }
 export async function runWebScraper({
@@ -71,6 +72,7 @@ export async function runWebScraper({
   team_id,
   bull_job_id,
   priority,
+  is_scrape=false,
 }: RunWebScraperParams): Promise<RunWebScraperResult> {
   try {
     const provider = new WebScraperDataProvider();
@@ -118,8 +120,8 @@ export async function runWebScraper({
         })
       : docs;
 
+    if(is_scrape === false) {
       const billingResult = await billTeam(team_id, filteredDocs.length);
-
       if (!billingResult.success) {
         // throw new Error("Failed to bill team, no subscription was found");
         return {
@@ -128,6 +130,9 @@ export async function runWebScraper({
           docs: [],
         };
       }
+    }
+
+    
 
     // This is where the returnvalue from the job is set
     onSuccess(filteredDocs, mode);

apps/api/src/types.ts

@@ -32,6 +32,7 @@ export interface WebScraperOptions {
   sitemapped?: boolean;
   webhook?: string;
   v1?: boolean;
+  is_scrape?: boolean;
 }
 
 export interface RunWebScraperParams {
@@ -46,6 +47,7 @@ export interface RunWebScraperParams {
   team_id: string;
   bull_job_id: string;
   priority?: number;
+  is_scrape?: boolean;
 }
 
 export interface RunWebScraperResult {

apps/js-sdk/firecrawl/types/index.d.ts

@@ -153,8 +153,6 @@ export interface ScrapeResponseV0 {
  * Includes options for both scraping and mapping during a crawl.
  */
 export interface CrawlParams {
-    scrapeOptions?: ScrapeParams;
-    crawlerOptions?: {
     includePaths?: string[];
     excludePaths?: string[];
     maxDepth?: number;
@@ -162,7 +160,7 @@ export interface CrawlParams {
     allowBackwardLinks?: boolean;
     allowExternalLinks?: boolean;
     ignoreSitemap?: boolean;
-    };
+    scrapeOptions?: ScrapeParams;
 }
 /**
  * Parameters for crawling operations on v0.