From 411d7f31c5347b1524632b6743e3b37508817b9a Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 3 Sep 2024 11:56:24 -0300 Subject: [PATCH] fix(sdks): fetch next/pagination --- apps/python-sdk/firecrawl/__init__.py | 2 +- apps/python-sdk/firecrawl/firecrawl.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py index 4b3807be..59c5fe35 100644 --- a/apps/python-sdk/firecrawl/__init__.py +++ b/apps/python-sdk/firecrawl/__init__.py @@ -13,7 +13,7 @@ import os from .firecrawl import FirecrawlApp -__version__ = "1.2.1" +__version__ = "1.2.2" # Define the logger for the Firecrawl project logger: logging.Logger = logging.getLogger("firecrawl") diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py index 75245e8d..d65b0341 100644 --- a/apps/python-sdk/firecrawl/firecrawl.py +++ b/apps/python-sdk/firecrawl/firecrawl.py @@ -346,6 +346,12 @@ class FirecrawlApp: status_data = status_response.json() if status_data['status'] == 'completed': if 'data' in status_data: + data = status_data['data'] + while 'next' in status_data: + status_response = self._get_request(status_data['next'], headers) + status_data = status_response.json() + data.extend(status_data['data']) + status_data['data'] = data return status_data else: raise Exception('Crawl job completed but no data was returned')