Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(sdks): fetch next/pagination

Browse Source
This commit is contained in:
rafaelsideguide
2024-09-03 11:56:24 -03:00
parent c3d90d494c
commit 411d7f31c5
2 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/python-sdk/firecrawl/__init__.py
+1 -1
View File
@@ -13,7 +13,7 @@ import os
from .firecrawl import FirecrawlApp from .firecrawl import FirecrawlApp
__version__ = "1.2.1" __version__ = "1.2.2"
# Define the logger for the Firecrawl project # Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl") logger: logging.Logger = logging.getLogger("firecrawl")
apps/python-sdk/firecrawl/firecrawl.py
+6
View File
@@ -346,6 +346,12 @@ class FirecrawlApp:
status_data = status_response.json() status_data = status_response.json()
if status_data['status'] == 'completed': if status_data['status'] == 'completed':
if 'data' in status_data: if 'data' in status_data:
data = status_data['data']
while 'next' in status_data:
status_response = self._get_request(status_data['next'], headers)
status_data = status_response.json()
data.extend(status_data['data'])
status_data['data'] = data
return status_data return status_data
else: else:
raise Exception('Crawl job completed but no data was returned') raise Exception('Crawl job completed but no data was returned')