Merge pull request #1034 from mendableai/sdk/fixed-none-undefined-on-response

[SDK] fixed none and undefined on response

apps/js-sdk/firecrawl/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "1.11.2",
+  "version": "1.11.3",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

apps/js-sdk/firecrawl/src/index.ts

@@ -565,23 +565,39 @@ export default class FirecrawlApp {
           if ("data" in statusData) {
             let data = statusData.data;
             while (typeof statusData === 'object' && 'next' in statusData) {
+              if (data.length === 0) {
+                break
+              }
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
             allData = data;
           }
         }
-        return ({
+
+        let resp: CrawlStatusResponse | ErrorResponse = {
           success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
-          next: response.data.next,
+          data: allData
-          data: allData,
+        }
-          error: response.data.error,
+
-        })
+        if (!response.data.success && response.data.error) {
+          resp = {
+            ...resp,
+            success: false,
+            error: response.data.error
+          } as ErrorResponse;
+        }
+
+        if (response.data.next) {
+          (resp as CrawlStatusResponse).next = response.data.next;
+        }
+        
+        return resp;
       } else {
         this.handleError(response, "check crawl status");
       }
@@ -799,23 +815,39 @@ export default class FirecrawlApp {
           if ("data" in statusData) {
             let data = statusData.data;
             while (typeof statusData === 'object' && 'next' in statusData) {
+              if (data.length === 0) {
+                break
+              }
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
             allData = data;
           }
         }
-        return ({
+
+        let resp: BatchScrapeStatusResponse | ErrorResponse = {
           success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
-          next: response.data.next,
+          data: allData
-          data: allData,
+        }
-          error: response.data.error,
+
-        })
+        if (!response.data.success && response.data.error) {
+          resp = {
+            ...resp,
+            success: false,
+            error: response.data.error
+          } as ErrorResponse;
+        }
+
+        if (response.data.next) {
+          (resp as BatchScrapeStatusResponse).next = response.data.next;
+        }
+        
+        return resp;
       } else {
         this.handleError(response, "check batch scrape status");
       }
@@ -971,6 +1003,9 @@ export default class FirecrawlApp {
               if ("data" in statusData) {
                 let data = statusData.data;
                 while (typeof statusData === 'object' && 'next' in statusData) {
+                  if (data.length === 0) {
+                    break
+                  }
                   statusResponse = await this.getRequest(statusData.next, headers);
                   statusData = statusResponse.data;
                   data = data.concat(statusData.data);

apps/python-sdk/firecrawl/__init__.py

@@ -13,7 +13,7 @@ import os
 
 from .firecrawl import FirecrawlApp # noqa
 
-__version__ = "1.8.0"
+__version__ = "1.8.1"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

apps/python-sdk/firecrawl/firecrawl.py

@@ -250,6 +250,8 @@ class FirecrawlApp:
                 if 'data' in status_data:
                     data = status_data['data']
                     while 'next' in status_data:
+                        if len(status_data['data']) == 0:
+                            break
                         next_url = status_data.get('next')
                         if not next_url:
                             logger.warning("Expected 'next' URL is missing.")
@@ -266,17 +268,25 @@ class FirecrawlApp:
                             logger.error(f"Error during pagination request: {e}")
                             break
                     status_data['data'] = data
-                    
+
-            return {
+            response = {
-                'success': True,
                 'status': status_data.get('status'),
                 'total': status_data.get('total'),
                 'completed': status_data.get('completed'),
                 'creditsUsed': status_data.get('creditsUsed'),
                 'expiresAt': status_data.get('expiresAt'),
-                'data': status_data.get('data'),
+                'data': status_data.get('data')
-                'error': status_data.get('error'),
+            }
-                'next': status_data.get('next', None)
+
+            if 'error' in status_data:
+                response['error'] = status_data['error']
+
+            if 'next' in status_data:
+                response['next'] = status_data['next']
+
+            return {
+                'success': False if 'error' in status_data else True,
+                **response
             }
         else:
             self._handle_error(response, 'check crawl status')
@@ -459,6 +469,8 @@ class FirecrawlApp:
                 if 'data' in status_data:
                     data = status_data['data']
                     while 'next' in status_data:
+                        if len(status_data['data']) == 0:
+                            break
                         next_url = status_data.get('next')
                         if not next_url:
                             logger.warning("Expected 'next' URL is missing.")
@@ -476,16 +488,24 @@ class FirecrawlApp:
                             break
                     status_data['data'] = data
 
-            return {
+            response = {
-                'success': True,
                 'status': status_data.get('status'),
                 'total': status_data.get('total'),
                 'completed': status_data.get('completed'),
                 'creditsUsed': status_data.get('creditsUsed'),
                 'expiresAt': status_data.get('expiresAt'),
-                'data': status_data.get('data'),
+                'data': status_data.get('data')
-                'error': status_data.get('error'),
+            }
-                'next': status_data.get('next', None)
+
+            if 'error' in status_data:
+                response['error'] = status_data['error']
+
+            if 'next' in status_data:
+                response['next'] = status_data['next']
+
+            return {
+                'success': False if 'error' in status_data else True,
+                **response
             }
         else:
             self._handle_error(response, 'check batch scrape status')
@@ -669,6 +689,8 @@ class FirecrawlApp:
                     if 'data' in status_data:
                         data = status_data['data']
                         while 'next' in status_data:
+                          if len(status_data['data']) == 0:
+                              break
                           status_response = self._get_request(status_data['next'], headers)
                           status_data = status_response.json()
                           data.extend(status_data.get('data', []))