4ab0893ffb
Adds a new browser management system with strategy pattern implementation: - Introduces BrowserManager class with strategy pattern support - Adds PlaywrightBrowserStrategy, CDPBrowserStrategy, and BuiltinBrowserStrategy - Implements BrowserProfileManager for profile management - Adds PagePoolConfig for browser page pooling - Includes comprehensive test suite for all browser strategies BREAKING CHANGE: Browser management has been moved to browser/ module. Direct usage of browser_manager.py and browser_profiler.py is deprecated.
51 lines
1.5 KiB
Python
51 lines
1.5 KiB
Python
import asyncio
|
|
from crawl4ai import (
|
|
AsyncWebCrawler,
|
|
BrowserConfig,
|
|
CrawlerRunConfig,
|
|
CacheMode,
|
|
DefaultMarkdownGenerator,
|
|
PruningContentFilter,
|
|
CrawlResult
|
|
)
|
|
|
|
async def example_cdp():
|
|
browser_conf = BrowserConfig(
|
|
headless=False,
|
|
cdp_url="http://localhost:9223"
|
|
)
|
|
crawler_config = CrawlerRunConfig(
|
|
session_id="test",
|
|
js_code = """(() => { return {"result": "Hello World!"} })()""",
|
|
js_only=True
|
|
)
|
|
async with AsyncWebCrawler(
|
|
config=browser_conf,
|
|
verbose=True,
|
|
) as crawler:
|
|
result : CrawlResult = await crawler.arun(
|
|
url="https://www.helloworld.org",
|
|
config=crawler_config,
|
|
)
|
|
print(result.js_execution_result)
|
|
|
|
|
|
async def main():
|
|
browser_config = BrowserConfig(headless=True, verbose=True)
|
|
async with AsyncWebCrawler(config=browser_config) as crawler:
|
|
crawler_config = CrawlerRunConfig(
|
|
cache_mode=CacheMode.BYPASS,
|
|
markdown_generator=DefaultMarkdownGenerator(
|
|
content_filter=PruningContentFilter(
|
|
threshold=0.48, threshold_type="fixed", min_word_threshold=0
|
|
)
|
|
),
|
|
)
|
|
result : CrawlResult = await crawler.arun(
|
|
url="https://www.helloworld.org", config=crawler_config
|
|
)
|
|
print(result.markdown.raw_markdown[:500])
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|