mirror of
https://github.com/mendableai/firecrawl.git
synced 2026-01-01 09:44:33 +00:00
21103b5a58
* fix: convert timeout from milliseconds to seconds in Python SDK - Fix timeout conversion in scrape_url method (line 596) - Fix timeout conversion in _post_request method (line 2207) - Add comprehensive tests for timeout functionality - Resolves issue #1848 The Python SDK was incorrectly passing timeout values in milliseconds directly to requests.post() which expects seconds, causing timeouts to be 1000x longer than intended (e.g. 60s became 16.6 hours). Co-Authored-By: rafael@sideguide.dev <rafael@sideguide.dev> * fix: handle timeout=0 edge case in conversion logic - Change condition from 'if timeout' to 'if timeout is not None' - Ensures timeout=0 is converted to 5.0 seconds instead of None - All timeout conversion tests now pass (5/5) Co-Authored-By: rafael@sideguide.dev <rafael@sideguide.dev> * feat: change default timeout from None to 30s (30000ms) - Update all timeout parameter defaults from None to 30000ms across SDK - ScrapeOptions, MapParams, and all method signatures now default to 30s - Update tests to verify new default timeout behavior (35s total with 5s buffer) - Add test for _post_request when no timeout key is present in data - Maintains backward compatibility for explicit timeout values - All 6 timeout conversion tests pass Co-Authored-By: rafael@sideguide.dev <rafael@sideguide.dev> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: rafael@sideguide.dev <rafael@sideguide.dev>
118 lines
3.9 KiB
Python
118 lines
3.9 KiB
Python
import unittest
|
|
from unittest.mock import patch, MagicMock
|
|
import os
|
|
from firecrawl import FirecrawlApp
|
|
|
|
|
|
class TestTimeoutConversion(unittest.TestCase):
|
|
|
|
@patch('requests.post')
|
|
def test_scrape_url_timeout_conversion(self, mock_post):
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_response.json.return_value = {
|
|
'success': True,
|
|
'data': {
|
|
'markdown': 'Test content'
|
|
}
|
|
}
|
|
mock_post.return_value = mock_response
|
|
|
|
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
|
|
app.scrape_url('https://example.com', timeout=60000)
|
|
|
|
args, kwargs = mock_post.call_args
|
|
self.assertEqual(kwargs['timeout'], 65.0)
|
|
|
|
@patch('requests.post')
|
|
def test_scrape_url_default_timeout(self, mock_post):
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_response.json.return_value = {
|
|
'success': True,
|
|
'data': {
|
|
'markdown': 'Test content'
|
|
}
|
|
}
|
|
mock_post.return_value = mock_response
|
|
|
|
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
|
|
app.scrape_url('https://example.com')
|
|
|
|
args, kwargs = mock_post.call_args
|
|
self.assertEqual(kwargs['timeout'], 35.0)
|
|
|
|
@patch('requests.post')
|
|
def test_post_request_timeout_conversion(self, mock_post):
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_post.return_value = mock_response
|
|
|
|
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
|
|
|
|
data = {'timeout': 30000}
|
|
headers = {'Content-Type': 'application/json'}
|
|
|
|
app._post_request('https://example.com/api', data, headers)
|
|
|
|
args, kwargs = mock_post.call_args
|
|
self.assertEqual(kwargs['timeout'], 35.0)
|
|
|
|
@patch('requests.post')
|
|
def test_post_request_default_timeout(self, mock_post):
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_post.return_value = mock_response
|
|
|
|
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
|
|
|
|
data = {'timeout': 30000, 'url': 'https://example.com'}
|
|
headers = {'Content-Type': 'application/json'}
|
|
|
|
app._post_request('https://example.com/api', data, headers)
|
|
|
|
args, kwargs = mock_post.call_args
|
|
self.assertEqual(kwargs['timeout'], 35.0)
|
|
|
|
@patch('requests.post')
|
|
def test_timeout_edge_cases(self, mock_post):
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_response.json.return_value = {
|
|
'success': True,
|
|
'data': {
|
|
'markdown': 'Test content'
|
|
}
|
|
}
|
|
mock_post.return_value = mock_response
|
|
|
|
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
|
|
|
|
app.scrape_url('https://example.com', timeout=1000)
|
|
args, kwargs = mock_post.call_args
|
|
self.assertEqual(kwargs['timeout'], 6.0)
|
|
|
|
app.scrape_url('https://example.com', timeout=0)
|
|
args, kwargs = mock_post.call_args
|
|
self.assertEqual(kwargs['timeout'], 5.0)
|
|
|
|
@patch('requests.post')
|
|
def test_post_request_no_timeout_key(self, mock_post):
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_post.return_value = mock_response
|
|
|
|
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
|
|
|
|
data = {'url': 'https://example.com'}
|
|
headers = {'Content-Type': 'application/json'}
|
|
|
|
app._post_request('https://example.com/api', data, headers)
|
|
|
|
args, kwargs = mock_post.call_args
|
|
self.assertIsNone(kwargs['timeout'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|