mirror of
https://github.com/unclecode/crawl4ai.git
synced 2025-11-03 21:42:57 +00:00
b957ff2ecd
- Add HTML attribute preservation in GoogleSearchCrawler - Fix lxml import references in utils.py - Remove unused ssl_certificate.json - Clean up imports and code organization in hub.py - Update test case formatting and remove unused image search test BREAKING CHANGE: Removed ssl_certificate.json file which might affect existing certificate validations
34 lines
1.0 KiB
Python
34 lines
1.0 KiB
Python
# test.py
|
|
from crawl4ai import CrawlerHub
|
|
import json
|
|
|
|
async def amazon_example():
|
|
if (crawler_cls := CrawlerHub.get("amazon_product")) :
|
|
crawler = crawler_cls()
|
|
print(f"Crawler version: {crawler_cls.meta['version']}")
|
|
print(f"Rate limits: {crawler_cls.meta.get('rate_limit', 'Unlimited')}")
|
|
print(await crawler.run("https://amazon.com/test"))
|
|
else:
|
|
print("Crawler not found!")
|
|
|
|
async def google_example():
|
|
# Get crawler dynamically
|
|
crawler_cls = CrawlerHub.get("google_search")
|
|
crawler = crawler_cls()
|
|
|
|
# Text search
|
|
text_results = await crawler.run(
|
|
query="apple inc",
|
|
search_type="text",
|
|
schema_cache_path="/Users/unclecode/.crawl4ai"
|
|
)
|
|
print(json.dumps(json.loads(text_results), indent=4))
|
|
|
|
# Image search
|
|
# image_results = await crawler.run(query="apple inc", search_type="image")
|
|
# print(image_results)
|
|
|
|
if __name__ == "__main__":
|
|
import asyncio
|
|
# asyncio.run(amazon_example())
|
|
asyncio.run(google_example()) |