mirror of
https://github.com/unclecode/crawl4ai.git
synced 2025-09-28 20:28:46 +00:00
72 lines
2.5 KiB
Python
72 lines
2.5 KiB
Python
|
import os
|
||
|
|
import sys
|
||
|
|
import pytest
|
||
|
|
import asyncio
|
||
|
|
import time
|
||
|
|
|
||
|
|
# Add the parent directory to the Python path
|
||
|
|
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
|
|
sys.path.append(parent_dir)
|
||
|
|
|
||
|
|
from crawl4ai.async_webcrawler import AsyncWebCrawler
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_crawl_speed():
|
||
|
|
async with AsyncWebCrawler(verbose=True) as crawler:
|
||
|
|
url = "https://www.nbcnews.com/business"
|
||
|
|
start_time = time.time()
|
||
|
|
result = await crawler.arun(url=url, bypass_cache=True)
|
||
|
|
end_time = time.time()
|
||
|
|
|
||
|
|
assert result.success
|
||
|
|
crawl_time = end_time - start_time
|
||
|
|
print(f"Crawl time: {crawl_time:.2f} seconds")
|
||
|
|
|
||
|
|
assert crawl_time < 10, f"Crawl took too long: {crawl_time:.2f} seconds"
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_concurrent_crawling_performance():
|
||
|
|
async with AsyncWebCrawler(verbose=True) as crawler:
|
||
|
|
urls = [
|
||
|
|
"https://www.nbcnews.com/business",
|
||
|
|
"https://www.example.com",
|
||
|
|
"https://www.python.org",
|
||
|
|
"https://www.github.com",
|
||
|
|
"https://www.stackoverflow.com"
|
||
|
|
]
|
||
|
|
|
||
|
|
start_time = time.time()
|
||
|
|
results = await crawler.arun_many(urls=urls, bypass_cache=True)
|
||
|
|
end_time = time.time()
|
||
|
|
|
||
|
|
total_time = end_time - start_time
|
||
|
|
print(f"Total time for concurrent crawling: {total_time:.2f} seconds")
|
||
|
|
|
||
|
|
assert all(result.success for result in results)
|
||
|
|
assert len(results) == len(urls)
|
||
|
|
|
||
|
|
assert total_time < len(urls) * 5, f"Concurrent crawling not significantly faster: {total_time:.2f} seconds"
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_crawl_speed_with_caching():
|
||
|
|
async with AsyncWebCrawler(verbose=True) as crawler:
|
||
|
|
url = "https://www.nbcnews.com/business"
|
||
|
|
|
||
|
|
start_time = time.time()
|
||
|
|
result1 = await crawler.arun(url=url, bypass_cache=True)
|
||
|
|
end_time = time.time()
|
||
|
|
first_crawl_time = end_time - start_time
|
||
|
|
|
||
|
|
start_time = time.time()
|
||
|
|
result2 = await crawler.arun(url=url, bypass_cache=False)
|
||
|
|
end_time = time.time()
|
||
|
|
second_crawl_time = end_time - start_time
|
||
|
|
|
||
|
|
assert result1.success and result2.success
|
||
|
|
print(f"First crawl time: {first_crawl_time:.2f} seconds")
|
||
|
|
print(f"Second crawl time (cached): {second_crawl_time:.2f} seconds")
|
||
|
|
|
||
|
|
assert second_crawl_time < first_crawl_time / 2, "Cached crawl not significantly faster"
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
pytest.main([__file__, "-v"])
|