yujunjun/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2025-11-02 19:12:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1034 from mendableai/sdk/fixed-none-undefined-on-response

Browse Source 
[SDK] fixed none and undefined on response
This commit is contained in:
Nicolas 2025-01-04 16:31:41 -03:00 committed by GitHub
commit ceb2104960
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 80 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
"version": "1.11.2",
"version": "1.11.3",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",

55
apps/js-sdk/firecrawl/src/index.ts
View File

@ -565,23 +565,39 @@ export default class FirecrawlApp {
if ("data" in statusData) {
let data = statusData.data;
while (typeof statusData === 'object' && 'next' in statusData) {
if (data.length === 0) {
break
}
statusData = (await this.getRequest(statusData.next, headers)).data;
data = data.concat(statusData.data);
}
allData = data;
}
}
return ({
let resp: CrawlStatusResponse | ErrorResponse = {
success: response.data.success,
status: response.data.status,
total: response.data.total,
completed: response.data.completed,
creditsUsed: response.data.creditsUsed,
expiresAt: new Date(response.data.expiresAt),
next: response.data.next,
data: allData,
error: response.data.error,
})
data: allData
}
if (!response.data.success && response.data.error) {
resp = {
...resp,
success: false,
error: response.data.error
} as ErrorResponse;
}
if (response.data.next) {
(resp as CrawlStatusResponse).next = response.data.next;
}
return resp;
} else {
this.handleError(response, "check crawl status");
}
@ -799,23 +815,39 @@ export default class FirecrawlApp {
if ("data" in statusData) {
let data = statusData.data;
while (typeof statusData === 'object' && 'next' in statusData) {
if (data.length === 0) {
break
}
statusData = (await this.getRequest(statusData.next, headers)).data;
data = data.concat(statusData.data);
}
allData = data;
}
}
return ({
let resp: BatchScrapeStatusResponse | ErrorResponse = {
success: response.data.success,
status: response.data.status,
total: response.data.total,
completed: response.data.completed,
creditsUsed: response.data.creditsUsed,
expiresAt: new Date(response.data.expiresAt),
next: response.data.next,
data: allData,
error: response.data.error,
})
data: allData
}
if (!response.data.success && response.data.error) {
resp = {
...resp,
success: false,
error: response.data.error
} as ErrorResponse;
}
if (response.data.next) {
(resp as BatchScrapeStatusResponse).next = response.data.next;
}
return resp;
} else {
this.handleError(response, "check batch scrape status");
}
@ -971,6 +1003,9 @@ export default class FirecrawlApp {
if ("data" in statusData) {
let data = statusData.data;
while (typeof statusData === 'object' && 'next' in statusData) {
if (data.length === 0) {
break
}
statusResponse = await this.getRequest(statusData.next, headers);
statusData = statusResponse.data;
data = data.concat(statusData.data);

2
apps/python-sdk/firecrawl/__init__.py
View File

@ -13,7 +13,7 @@ import os
from .firecrawl import FirecrawlApp # noqa
__version__ = "1.8.0"
__version__ = "1.8.1"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")

44
apps/python-sdk/firecrawl/firecrawl.py
View File

@ -250,6 +250,8 @@ class FirecrawlApp:
if 'data' in status_data:
data = status_data['data']
while 'next' in status_data:
if len(status_data['data']) == 0:
break
next_url = status_data.get('next')
if not next_url:
logger.warning("Expected 'next' URL is missing.")
@ -266,17 +268,25 @@ class FirecrawlApp:
logger.error(f"Error during pagination request: {e}")
break
status_data['data'] = data
return {
'success': True,
response = {
'status': status_data.get('status'),
'total': status_data.get('total'),
'completed': status_data.get('completed'),
'creditsUsed': status_data.get('creditsUsed'),
'expiresAt': status_data.get('expiresAt'),
'data': status_data.get('data'),
'error': status_data.get('error'),
'next': status_data.get('next', None)
'data': status_data.get('data')
}
if 'error' in status_data:
response['error'] = status_data['error']
if 'next' in status_data:
response['next'] = status_data['next']
return {
'success': False if 'error' in status_data else True,
**response
}
else:
self._handle_error(response, 'check crawl status')
@ -459,6 +469,8 @@ class FirecrawlApp:
if 'data' in status_data:
data = status_data['data']
while 'next' in status_data:
if len(status_data['data']) == 0:
break
next_url = status_data.get('next')
if not next_url:
logger.warning("Expected 'next' URL is missing.")
@ -476,16 +488,24 @@ class FirecrawlApp:
break
status_data['data'] = data
return {
'success': True,
response = {
'status': status_data.get('status'),
'total': status_data.get('total'),
'completed': status_data.get('completed'),
'creditsUsed': status_data.get('creditsUsed'),
'expiresAt': status_data.get('expiresAt'),
'data': status_data.get('data'),
'error': status_data.get('error'),
'next': status_data.get('next', None)
'data': status_data.get('data')
}
if 'error' in status_data:
response['error'] = status_data['error']
if 'next' in status_data:
response['next'] = status_data['next']
return {
'success': False if 'error' in status_data else True,
**response
}
else:
self._handle_error(response, 'check batch scrape status')
@ -669,6 +689,8 @@ class FirecrawlApp:
if 'data' in status_data:
data = status_data['data']
while 'next' in status_data:
if len(status_data['data']) == 0:
break
status_response = self._get_request(status_data['next'], headers)
status_data = status_response.json()
data.extend(status_data.get('data', []))