yujunjun/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: move from faust-cchardet to chardet

Browse Source
This commit is contained in:
Aravind Karnam 2025-04-03 17:42:51 +05:30
parent 4133e5460d
commit 7155778eac
3 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
crawl4ai/async_crawler_strategy.py
View File

@ -24,7 +24,7 @@ from .browser_manager import BrowserManager
import aiofiles import aiofiles
import aiohttp import aiohttp
import cchardet import chardet
from aiohttp.client import ClientTimeout from aiohttp.client import ClientTimeout
from urllib.parse import urlparse from urllib.parse import urlparse
from types import MappingProxyType from types import MappingProxyType
@ -1822,7 +1822,7 @@ class AsyncHTTPCrawlerStrategy(AsyncCrawlerStrategy):
encoding = response.charset encoding = response.charset
if not encoding: if not encoding:
encoding = cchardet.detect(content.tobytes())['encoding'] or 'utf-8' encoding = chardet.detect(content.tobytes())['encoding'] or 'utf-8'
result = AsyncCrawlResponse( result = AsyncCrawlResponse(
html=content.tobytes().decode(encoding, errors='replace'), html=content.tobytes().decode(encoding, errors='replace'),

2
pyproject.toml
View File

@ -40,7 +40,7 @@ dependencies = [
"fake-useragent>=2.0.3", "fake-useragent>=2.0.3",
"click>=8.1.7", "click>=8.1.7",
"pyperclip>=1.8.2", "pyperclip>=1.8.2",
"faust-cchardet>=2.1.19", "chardet>=5.2.0",
"aiohttp>=3.11.11", "aiohttp>=3.11.11",
"brotli>=1.1.0", "brotli>=1.1.0",
"humanize>=4.10.0", "humanize>=4.10.0",

2
requirements.txt
View File

@ -21,5 +21,5 @@ psutil>=6.1.1
nltk>=3.9.1 nltk>=3.9.1
rich>=13.9.4 rich>=13.9.4
cssselect>=1.2.0 cssselect>=1.2.0
faust-cchardet>=2.1.19 chardet>=5.2.0
brotli>=1.1.0 brotli>=1.1.0