feat(bulk/scrape): add node and python SDK integration + docs

2024-10-22 18:58:48 +02:00
parent 03b37998fd
commit 3cd328cf93
4 changed files with 358 additions and 0 deletions
@@ -145,6 +145,46 @@ watch.addEventListener("done", state => {
 });
 ```

+### Bulk scraping multiple URLs
+
+To bulk scrape multiple URLs with error handling, use the `bulkScrapeUrls` method. It takes the starting URLs and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the output formats.
+
+```js
+const bulkScrapeResponse = await app.bulkScrapeUrls(['https://firecrawl.dev', 'https://mendable.ai'], {
+  formats: ['markdown', 'html'],
+})
+```
+
+
+#### Asynchronous bulk scrape
+
+To initiate an asynchronous bulk scrape, utilize the `asyncBulkScrapeUrls` method. This method requires the starting URLs and optional parameters as inputs. The params argument enables you to define various settings for the scrape, such as the output formats. Upon successful initiation, this method returns an ID, which is essential for subsequently checking the status of the bulk scrape.
+
+```js
+const asyncBulkScrapeResult = await app.asyncBulkScrapeUrls(['https://firecrawl.dev', 'https://mendable.ai'], { formats: ['markdown', 'html'] });
+```
+
+#### Bulk scrape with WebSockets
+
+To use bulk scrape with WebSockets, use the `bulkScrapeUrlsAndWatch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the bulk scrape job, such as the output formats.
+
+```js
+// Bulk scrape multiple URLs with WebSockets:
+const watch = await app.bulkScrapeUrlsAndWatch(['https://firecrawl.dev', 'https://mendable.ai'], { formats: ['markdown', 'html'] });
+
+watch.addEventListener("document", doc => {
+ console.log("DOC", doc.detail);
+});
+
+watch.addEventListener("error", err => {
+ console.error("ERR", err.detail.error);
+});
+
+watch.addEventListener("done", state => {
+ console.log("DONE", state.detail.status);
+});
+```
+
 ## Error Handling

 The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message. The examples above demonstrate how to handle these errors using `try/catch` blocks.
@@ -493,6 +493,144 @@ export default class FirecrawlApp {
    return { success: false, error: "Internal server error." };
  }

+  /**
+   * Initiates a bulk scrape job for multiple URLs using the Firecrawl API.
+   * @param url - The URLs to scrape.
+   * @param params - Additional parameters for the scrape request.
+   * @param pollInterval - Time in seconds for job status checks.
+   * @param idempotencyKey - Optional idempotency key for the request.
+   * @returns The response from the crawl operation.
+   */
+  async bulkScrapeUrls(
+    urls: string[],
+    params?: ScrapeParams,
+    pollInterval: number = 2,
+    idempotencyKey?: string
+  ): Promise<CrawlStatusResponse | ErrorResponse> {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData: any = { urls, ...(params ?? {}) };
+    try {
+      const response: AxiosResponse = await this.postRequest(
+        this.apiUrl + `/v1/bulk/scrape`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        const id: string = response.data.id;
+        return this.monitorJobStatus(id, headers, pollInterval);
+      } else {
+        this.handleError(response, "start bulk scrape job");
+      }
+    } catch (error: any) {
+      if (error.response?.data?.error) {
+        throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+      } else {
+        throw new FirecrawlError(error.message, 500);
+      }
+    }
+    return { success: false, error: "Internal server error." };
+  }
+
+  async asyncBulkScrapeUrls(
+    urls: string[],
+    params?: ScrapeParams,
+    idempotencyKey?: string
+  ): Promise<CrawlResponse | ErrorResponse> {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData: any = { urls, ...(params ?? {}) };
+    try {
+      const response: AxiosResponse = await this.postRequest(
+        this.apiUrl + `/v1/bulk/scrape`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "start bulk scrape job");
+      }
+    } catch (error: any) {
+      if (error.response?.data?.error) {
+        throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+      } else {
+        throw new FirecrawlError(error.message, 500);
+      }
+    }
+    return { success: false, error: "Internal server error." };
+  }
+
+  /**
+   * Initiates a bulk scrape job and returns a CrawlWatcher to monitor the job via WebSocket.
+   * @param urls - The URL to scrape.
+   * @param params - Additional parameters for the scrape request.
+   * @param idempotencyKey - Optional idempotency key for the request.
+   * @returns A CrawlWatcher instance to monitor the crawl job.
+   */
+  async bulkScrapeUrlsAndWatch(
+    urls: string[],
+    params?: ScrapeParams,
+    idempotencyKey?: string,
+  ) {
+    const crawl = await this.asyncBulkScrapeUrls(urls, params, idempotencyKey);
+
+    if (crawl.success && crawl.id) {
+      const id = crawl.id;
+      return new CrawlWatcher(id, this);
+    }
+
+    throw new FirecrawlError("Bulk scrape job failed to start", 400);
+  }
+
+  /**
+   * Checks the status of a bulk scrape job using the Firecrawl API.
+   * @param id - The ID of the bulk scrape operation.
+   * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @returns The response containing the job status.
+   */
+  async checkBulkScrapeStatus(id?: string, getAllData = false): Promise<CrawlStatusResponse | ErrorResponse> {
+    if (!id) {
+      throw new FirecrawlError("No bulk scrape ID provided", 400);
+    }
+
+    const headers: AxiosRequestHeaders = this.prepareHeaders();
+    try {
+      const response: AxiosResponse = await this.getRequest(
+        `${this.apiUrl}/v1/bulk/scrape/${id}`,
+        headers
+      );
+      if (response.status === 200) {
+        let allData = response.data.data;
+        if (getAllData && response.data.status === "completed") {
+          let statusData = response.data
+          if ("data" in statusData) {
+            let data = statusData.data;
+            while ('next' in statusData) {
+              statusData = (await this.getRequest(statusData.next, headers)).data;
+              data = data.concat(statusData.data);
+            }
+            allData = data;
+          }
+        }
+        return ({
+          success: response.data.success,
+          status: response.data.status,
+          total: response.data.total,
+          completed: response.data.completed,
+          creditsUsed: response.data.creditsUsed,
+          expiresAt: new Date(response.data.expiresAt),
+          next: response.data.next,
+          data: allData,
+          error: response.data.error,
+        })
+      } else {
+        this.handleError(response, "check bulk scrape status");
+      }
+    } catch (error: any) {
+      throw new FirecrawlError(error.message, 500);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+
  /**
   * Prepares the headers for an API request.
   * @param idempotencyKey - Optional key to ensure idempotency.