feat(deep-crawling): add DFS strategy and update exports; refactor CLI entry point

This commit is contained in:
UncleCode 2025-02-09 20:23:40 +08:00
parent 19df96ed56
commit 467be9ac76
5 changed files with 9 additions and 3 deletions

View File

@ -33,9 +33,10 @@ from .async_dispatcher import (
)
from .docker_client import Crawl4aiDockerClient
from .hub import CrawlerHub
from .deep_crawling import DeepCrawlStrategy
__all__ = [
"AsyncWebCrawler",
"DeepCrawlStrategy",
"CrawlResult",
"CrawlerHub",
"CacheMode",

0
crawl4ai/cli.py Normal file
View File

View File

@ -2,10 +2,12 @@
from .base_strategy import DeepCrawlDecorator, DeepCrawlStrategy
from .bfs_strategy import BFSDeepCrawlStrategy
from .bff_strategy import BestFirstCrawlingStrategy
from .dfs_strategy import DFSDeepCrawlStrategy
__all__ = [
"DeepCrawlDecorator",
"DeepCrawlStrategy",
"BFSDeepCrawlStrategy",
"BestFirstCrawlingStrategy",
"DFSDeepCrawlStrategy",
]

View File

@ -224,6 +224,7 @@ def create_common_filter_chain() -> FilterChain:
# Use __slots__ and array for maximum memory/speed efficiency
@dataclass
class FastFilterStats:
__slots__ = ("_counters",)

View File

@ -37,7 +37,9 @@ dependencies = [
"rich>=13.9.4",
"cssselect>=1.2.0",
"httpx==0.27.2",
"fake-useragent>=2.0.3"
"fake-useragent>=2.0.3",
"click>=8.1.7",
"pyperclip>=1.8.2"
]
classifiers = [
"Development Status :: 4 - Beta",
@ -70,7 +72,7 @@ crawl4ai-download-models = "crawl4ai.model_loader:main"
crawl4ai-migrate = "crawl4ai.migrations:main"
crawl4ai-setup = "crawl4ai.install:post_install"
crawl4ai-doctor = "crawl4ai.install:doctor"
crawl = "crawl4ai.cli:cli"
crwl = "crawl4ai.cli:cli"
[tool.setuptools]
packages = {find = {where = ["."], include = ["crawl4ai*"]}}