fix(sdks): fetch next/pagination

2024-09-03 11:56:24 -03:00
parent c3d90d494c
commit 411d7f31c5
2 changed files with 7 additions and 1 deletions
@@ -346,6 +346,12 @@ class FirecrawlApp:
                status_data = status_response.json()
                if status_data['status'] == 'completed':
                    if 'data' in status_data:
+                        data = status_data['data']
+                        while 'next' in status_data:
+                          status_response = self._get_request(status_data['next'], headers)
+                          status_data = status_response.json()
+                          data.extend(status_data['data'])
+                        status_data['data'] = data
                        return status_data
                    else:
                        raise Exception('Crawl job completed but no data was returned')