diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index c488843..685caeb 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -1,5 +1,3 @@ -from email import header -from re import I from .config import ( MIN_WORD_THRESHOLD, IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD, @@ -23,7 +21,7 @@ import inspect from typing import Any, Dict, Optional from enum import Enum -def to_serializable_dict(obj: Any) -> Dict: +def to_serializable_dict(obj: Any, ignore_default_value : bool = False) -> Dict: """ Recursively convert an object to a serializable dictionary using {type, params} structure for complex objects. @@ -60,7 +58,9 @@ def to_serializable_dict(obj: Any) -> Dict: "type": "dict", # Mark as plain dictionary "value": {str(k): to_serializable_dict(v) for k, v in obj.items()} } - + + _type = obj.__class__.__name__ + # Handle class instances if hasattr(obj, '__class__'): # Get constructor signature @@ -77,10 +77,18 @@ def to_serializable_dict(obj: Any) -> Dict: # Only include if different from default, considering empty values if not (is_empty_value(value) and is_empty_value(param.default)): - if value != param.default: + if value != param.default and not ignore_default_value: current_values[name] = to_serializable_dict(value) - _type = obj.__class__.__name__ + if hasattr(obj, '__slots__'): + for slot in obj.__slots__: + if slot.startswith('_'): # Handle private slots + attr_name = slot[1:] # Remove leading '_' + value = getattr(obj, slot, None) + if value is not None: + current_values[attr_name] = to_serializable_dict(value) + + return { "type": obj.__class__.__name__, @@ -169,6 +177,8 @@ class BrowserConfig(): If None, no additional proxy config. Default: None. viewport_width (int): Default viewport width for pages. Default: 1080. viewport_height (int): Default viewport height for pages. Default: 600. + viewport (dict): Default viewport dimensions for pages. If set, overrides viewport_width and viewport_height. + Default: None. verbose (bool): Enable verbose logging. Default: True. accept_downloads (bool): Whether to allow file downloads. If True, requires a downloads_path. @@ -211,6 +221,7 @@ class BrowserConfig(): proxy_config: dict = None, viewport_width: int = 1080, viewport_height: int = 600, + viewport: dict = None, accept_downloads: bool = False, downloads_path: str = None, storage_state : Union[str, dict, None]=None, @@ -249,6 +260,10 @@ class BrowserConfig(): self.proxy_config = proxy_config self.viewport_width = viewport_width self.viewport_height = viewport_height + self.viewport = viewport + if self.viewport is not None: + self.viewport_width = self.viewport.get("width", 1080) + self.viewport_height = self.viewport.get("height", 600) self.accept_downloads = accept_downloads self.downloads_path = downloads_path self.storage_state = storage_state @@ -436,6 +451,13 @@ class HTTPCrawlerConfig(): return HTTPCrawlerConfig.from_kwargs(config) class CrawlerRunConfig(): + _UNWANTED_PROPS = { + 'disable_cache' : 'Instead, use cache_mode=CacheMode.DISABLED', + 'bypass_cache' : 'Instead, use cache_mode=CacheMode.BYPASS', + 'no_cache_read' : 'Instead, use cache_mode=CacheMode.WRITE_ONLY', + 'no_cache_write' : 'Instead, use cache_mode=CacheMode.READ_ONLY', + } + """ Configuration class for controlling how the crawler runs each crawl operation. This includes parameters for content extraction, page manipulation, waiting conditions, @@ -680,6 +702,7 @@ class CrawlerRunConfig(): deep_crawl_strategy: Optional[DeepCrawlStrategy] = None, ): + # TODO: Planning to set properties dynamically based on the __init__ signature self.url = url # Content Processing Parameters @@ -791,6 +814,24 @@ class CrawlerRunConfig(): # Deep Crawl Parameters self.deep_crawl_strategy = deep_crawl_strategy + + def __getattr__(self, name): + """Handle attribute access.""" + if name in self._UNWANTED_PROPS: + raise AttributeError(f"Getting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}") + raise AttributeError(f"'{self.__class__.__name__}' has no attribute '{name}'") + + def __setattr__(self, name, value): + """Handle attribute setting.""" + # TODO: Planning to set properties dynamically based on the __init__ signature + sig = inspect.signature(self.__init__) + all_params = sig.parameters # Dictionary of parameter names and their details + + if name in self._UNWANTED_PROPS and value is not all_params[name].default: + raise AttributeError(f"Setting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}") + + super().__setattr__(name, value) + @staticmethod def from_kwargs(kwargs: dict) -> "CrawlerRunConfig": return CrawlerRunConfig( @@ -988,3 +1029,5 @@ class CrawlerRunConfig(): config_dict = self.to_dict() config_dict.update(kwargs) return CrawlerRunConfig.from_kwargs(config_dict) + + diff --git a/crawl4ai/deep_crawling/crazy.py b/crawl4ai/deep_crawling/crazy.py index 8804e7b..d2bc27e 100644 --- a/crawl4ai/deep_crawling/crazy.py +++ b/crawl4ai/deep_crawling/crazy.py @@ -10,7 +10,7 @@ import inspect from crawl4ai import CacheMode from crawl4ai.async_configs import CrawlerRunConfig from crawl4ai.models import CrawlResult, TraversalStats -from crawl4ai.deep_crawling.filters import FastFilterChain +from crawl4ai.deep_crawling.filters import FilterChain from crawl4ai.async_webcrawler import AsyncWebCrawler import time import logging @@ -313,7 +313,7 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy): def __init__(self, max_depth: int, - filter_chain: FastFilterChain = FastFilterChain(), + filter_chain: FilterChain = FilterChain(), priority_fn: Callable[[str], Awaitable[float]] = lambda url: 1.0, logger: logging.Logger = None): self.max_depth = max_depth @@ -408,7 +408,7 @@ async def main(): strategy = BFSDeepCrawlStrategy( max_depth=2, priority_fn=lambda url: 1.0 / (len(url) + 1e-9), # Inverse length priority - # filter_chain=FastFilterChain(...) + # filter_chain=FilterChain(...) ) config: CrawlerRunConfig = CrawlerRunConfig( diff --git a/deploy/docker/README.md b/deploy/docker/README.md index f62e58c..fbed657 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -469,57 +469,40 @@ print(config.dump()) # Use this JSON in your API calls "type": "BFSDeepCrawlStrategy", "params": { "max_depth": 3, - "max_pages": 100, "filter_chain": { - "type": "FastFilterChain", + "type": "FilterChain", "params": { "filters": [ { - "type": "FastContentTypeFilter", + "type": "ContentTypeFilter", "params": { "allowed_types": ["text/html", "application/xhtml+xml"] } }, { - "type": "FastDomainFilter", + "type": "DomainFilter", "params": { "allowed_domains": ["blog.*", "docs.*"], - "blocked_domains": ["ads.*", "analytics.*"] - } - }, - { - "type": "FastURLPatternFilter", - "params": { - "allowed_patterns": ["^/blog/", "^/docs/"], - "blocked_patterns": [".*/ads/", ".*/sponsored/"] } } ] } }, "url_scorer": { - "type": "FastCompositeScorer", + "type": "CompositeScorer", "params": { "scorers": [ { - "type": "FastKeywordRelevanceScorer", + "type": "KeywordRelevanceScorer", "params": { "keywords": ["tutorial", "guide", "documentation"], - "weight": 1.0 } }, { - "type": "FastPathDepthScorer", + "type": "PathDepthScorer", "params": { "weight": 0.5, - "preferred_depth": 2 - } - }, - { - "type": "FastFreshnessScorer", - "params": { - "weight": 0.8, - "max_age_days": 365 + "optimal_depth": 3 } } ] diff --git a/docs/examples/docker_config_obj.py b/docs/examples/docker_config_obj.py new file mode 100644 index 0000000..6ddf157 --- /dev/null +++ b/docs/examples/docker_config_obj.py @@ -0,0 +1,249 @@ +from crawl4ai import BrowserConfig, CrawlerRunConfig, PruningContentFilter, DefaultMarkdownGenerator +from crawl4ai.deep_crawling.filters import ContentTypeFilter, DomainFilter +from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer, PathDepthScorer +from crawl4ai.cache_context import CacheMode +from crawl4ai.deep_crawling.bfs_strategy import BFSDeepCrawlStrategy +from crawl4ai.deep_crawling.filters import FilterChain +from crawl4ai.deep_crawling.scorers import CompositeScorer +from crawl4ai.docker_client import Crawl4aiDockerClient +import json +from rich.console import Console +from rich.syntax import Syntax + +console = Console() + +def print_json(data: dict, title: str = None): + """Helper to print JSON prettily with syntax highlighting""" + if title: + console.print(f"\n[bold blue]{title}[/bold blue]") + json_str = json.dumps(data, indent=2) + syntax = Syntax(json_str, "json", theme="monokai", line_numbers=True) + console.print(syntax) + +async def part1_basic_config(): + """PART 1: Understanding Basic Configuration Objects + + Here we create simple configuration objects and examine their structure. + This helps understand the basic type-params pattern used throughout the API. + """ + console.print("\n[bold green]Explanation:[/bold green] Configuration objects like BrowserConfig and CrawlerRunConfig are the foundation of Crawl4AI. They define how the crawler behaves—e.g., whether it runs headless or how it processes content. These objects use a 'type-params' pattern: 'type' identifies the object class, and 'params' holds its settings. This structure is key because it’s reusable and can be serialized into JSON for API calls.") + + # Create a simple browser config + browser_config = BrowserConfig( + headless=False, + viewport_width=500, + headers = {"User-Agent": "Mozilla/5.0"} + ) + + # Show its structure + print_json(browser_config.dump(), "Simple Browser Config Structure") + + # Create a more complex config with nested objects + crawler_config = CrawlerRunConfig( + word_count_threshold=200, + markdown_generator=DefaultMarkdownGenerator( + content_filter=PruningContentFilter(threshold=0.5) + ) + ) + + print_json(crawler_config.dump(), "Complex Config with Nested Objects") + +async def part2_manual_json(): + """PART 2: Building JSON Manually + + Learn how to construct the JSON structure by hand. + This demonstrates deep understanding of the configuration format. + """ + console.print("\n[bold green]Explanation:[/bold green] Manually building JSON configurations mirrors how the API expects data. It’s a hands-on way to learn the exact structure—each object has a 'type' and 'params' section. This is useful when you’re troubleshooting or working without the SDK, as it forces you to understand every detail of the config format.") + + # Manual browser config + manual_browser = { + "type": "BrowserConfig", + "params": { + "headless": True, + "viewport": { + "type": "dict", + "value": { + "width": 1200, + "height": 800 + } + } + } + } + + # Validate by loading into BrowserConfig + loaded_config = BrowserConfig.load(manual_browser) + print_json(loaded_config.dump(), "Manually Created -> Loaded -> Dumped") + + # Show they're equivalent + original = BrowserConfig(headless=True, viewport={"width": 1200, "height": 800}) + assert loaded_config.dump() == original.dump(), "Configs are equivalent!" + +async def part3_complex_structures(): + """PART 3: Working with Complex Nested Structures + + Explore more complex configurations with multiple levels of nesting. + This shows how the type-params pattern scales to complex scenarios. + """ + console.print("\n[bold green]Explanation:[/bold green] Real-world crawling often requires detailed settings—like filtering content or customizing output. Here, we nest objects (e.g., a markdown generator with a content filter) using the same 'type-params' pattern. This nesting lets you fine-tune the crawler’s behavior at multiple levels, making it powerful and flexible.") + + config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + markdown_generator=DefaultMarkdownGenerator( + content_filter=PruningContentFilter() + ), + deep_crawl_strategy=BFSDeepCrawlStrategy( + max_depth=5, + filter_chain=FilterChain( + filters=[ + ContentTypeFilter(allowed_types=["text/html"]), + DomainFilter(allowed_domains=["example.com"]) + ] + ), + url_scorer=CompositeScorer( + scorers=[ + KeywordRelevanceScorer(keywords=["data", "analysis"]), + PathDepthScorer(optimal_depth=3) + ] + ) + ) + ) + + print_json(config.dump(), "Deep Nested Configuration") + +async def part4_client_sdk(): + """PART 4: Using the Client SDK + + Demonstrate how the SDK makes working with the API simple by handling + all the complex serialization automatically. + """ + console.print("\n[bold green]Explanation:[/bold green] The Crawl4aiDockerClient SDK is a time-saver—it takes your configuration objects and turns them into API-ready JSON automatically. This means less manual work and fewer mistakes. You just define your settings, pass them to the SDK, and it handles the rest, making crawling easier and faster.") + + async with Crawl4aiDockerClient(base_url="http://localhost:8000") as client: + # You would normally authenticate here if JWT is enabled + await client.authenticate("user@example.com") + + # Create configs + browser_config = BrowserConfig(headless=True) + crawler_config = CrawlerRunConfig(stream=False) + + # SDK handles all serialization + result = await client.crawl( + urls=["https://example.com"], + browser_config=browser_config, + crawler_config=crawler_config + ) + + console.print("\n[bold green]🚀 Crawl completed successfully![/bold green]") + console.print(f"Markdown length: {len(result.markdown)} characters") + +async def part5_direct_api(): + """PART 5: Using the API Directly + + Learn how to make direct API calls without the SDK. + This demonstrates the raw request structure and gives more control. + """ + console.print("\n[bold green]Explanation:[/bold green] Skipping the SDK means you’re in full control—you build the JSON payload yourself and send it to the API. This is harder but gives you a deeper understanding of how Crawl4AI works under the hood. It’s also useful if you’re integrating with systems that don’t use the SDK.") + + import aiohttp + from datetime import datetime + + # Prepare the request payload + payload = { + "urls": ["https://example.com"], + "browser_config": { + "type": "BrowserConfig", + "params": { + "headless": True, + "viewport": { + "type": "dict", + "value": { + "width": 1200, + "height": 800 + } + } + } + }, + "crawler_config": { + "type": "CrawlerRunConfig", + "params": { + "cache_mode": "bypass", + "markdown_generator": { + "type": "DefaultMarkdownGenerator", + "params": { + "content_filter": { + "type": "PruningContentFilter", + "params": { + "threshold": 0.48, + "threshold_type": "fixed" + } + } + } + } + } + } + } + + print_json(payload, "Direct API Request Payload") + + async with aiohttp.ClientSession() as session: + # If JWT is enabled, get token first + token_response = await session.post( + "http://localhost:8000/token", + json={"email": "user@example.com"} + ) + token = (await token_response.json())["access_token"] + headers = {"Authorization": f"Bearer {token}"} + + # Make the crawl request + start_time = datetime.now() + async with session.post( + "http://localhost:8000/crawl", + json=payload, + headers=headers # comment if using JWT + ) as response: + result = await response.json() + duration = (datetime.now() - start_time).total_seconds() + + console.print(f"\n[bold green]✅ API call completed in {duration:.2f}s[/bold green]") + print_json(result, "API Response") + +async def part6_wrap_up(): + """PART 6: Wrap-Up and Key Takeaways + + Summarize the key concepts learned in this tutorial. + """ + console.print("\n[bold yellow]🎓 Tutorial Wrap-Up[/bold yellow]") + console.print("[italic]Key Takeaways:[/italic]\n") + console.print("- **Configurations:** Use the type-params pattern to define settings flexibly.") + console.print("- **Manual JSON:** Build configs by hand to master the structure.") + console.print("- **Nesting:** Customize deeply with nested objects.") + console.print("- **SDK:** Simplify API calls with automatic serialization.") + console.print("- **Direct API:** Gain control by crafting raw requests.") + console.print("\n[bold green]🚀 You’re ready to crawl with Crawl4AI![/bold green]") + +async def main(): + """Main tutorial runner that executes each part in sequence""" + console.print("\n[bold yellow]🎓 Crawl4AI Docker Tutorial[/bold yellow]") + console.print("[italic]Learn how to work with configuration objects and the Docker API[/italic]\n") + + parts = [ + (part1_basic_config, "Understanding Basic Configurations"), + (part2_manual_json, "Manual JSON Construction"), + (part3_complex_structures, "Complex Nested Structures"), + (part4_client_sdk, "Using the Client SDK"), + (part5_direct_api, "Direct API Integration"), + (part6_wrap_up, "Wrap-Up and Key Takeaways") + ] + + for func, title in parts: + console.print(f"\n[bold cyan]📚 {title}[/bold cyan]") + console.print("[dim]" + func.__doc__.strip() + "[/dim]\n") + await func() + if func != part6_wrap_up: # No pause after wrap-up + input("\nPress Enter to continue...\n") + +# Run the tutorial +if __name__ == "__main__": + import asyncio + asyncio.run(main()) \ No newline at end of file diff --git a/docs/examples/docker_python_rest_api_example.py b/docs/examples/docker_python_rest_api.py similarity index 100% rename from docs/examples/docker_python_rest_api_example.py rename to docs/examples/docker_python_rest_api.py diff --git a/docs/examples/docker_python_sdk_example.py b/docs/examples/docker_python_sdk.py similarity index 100% rename from docs/examples/docker_python_sdk_example.py rename to docs/examples/docker_python_sdk.py diff --git a/docs/examples/extraction_strategies_example.py b/docs/examples/extraction_strategies_examples.py similarity index 100% rename from docs/examples/extraction_strategies_example.py rename to docs/examples/extraction_strategies_examples.py