a58c8000aa
Replace crawler_manager.py with simpler crawler_pool.py implementation: - Add global page semaphore for hard concurrency cap - Implement browser pool with idle cleanup - Add playground UI for testing and stress testing - Update API handlers to use pooled crawlers - Enhance logging levels and symbols BREAKING CHANGE: Removes CrawlerManager class in favor of simpler pool-based approach
61 lines
2.3 KiB
Python
61 lines
2.3 KiB
Python
# crawler_pool.py (new file)
|
||
import asyncio, json, hashlib, time, psutil
|
||
from contextlib import suppress
|
||
from typing import Dict
|
||
from crawl4ai import AsyncWebCrawler, BrowserConfig
|
||
from typing import Dict
|
||
from utils import load_config
|
||
|
||
CONFIG = load_config()
|
||
|
||
POOL: Dict[str, AsyncWebCrawler] = {}
|
||
LAST_USED: Dict[str, float] = {}
|
||
LOCK = asyncio.Lock()
|
||
|
||
MEM_LIMIT = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0) # % RAM – refuse new browsers above this
|
||
IDLE_TTL = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 1800) # close if unused for 30 min
|
||
|
||
def _sig(cfg: BrowserConfig) -> str:
|
||
payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",",":"))
|
||
return hashlib.sha1(payload.encode()).hexdigest()
|
||
|
||
async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
|
||
try:
|
||
sig = _sig(cfg)
|
||
async with LOCK:
|
||
if sig in POOL:
|
||
LAST_USED[sig] = time.time();
|
||
return POOL[sig]
|
||
if psutil.virtual_memory().percent >= MEM_LIMIT:
|
||
raise MemoryError("RAM pressure – new browser denied")
|
||
crawler = AsyncWebCrawler(config=cfg, thread_safe=False)
|
||
await crawler.start()
|
||
POOL[sig] = crawler; LAST_USED[sig] = time.time()
|
||
return crawler
|
||
except MemoryError as e:
|
||
raise MemoryError(f"RAM pressure – new browser denied: {e}")
|
||
except Exception as e:
|
||
raise RuntimeError(f"Failed to start browser: {e}")
|
||
finally:
|
||
if sig in POOL:
|
||
LAST_USED[sig] = time.time()
|
||
else:
|
||
# If we failed to start the browser, we should remove it from the pool
|
||
POOL.pop(sig, None)
|
||
LAST_USED.pop(sig, None)
|
||
# If we failed to start the browser, we should remove it from the pool
|
||
async def close_all():
|
||
async with LOCK:
|
||
await asyncio.gather(*(c.close() for c in POOL.values()), return_exceptions=True)
|
||
POOL.clear(); LAST_USED.clear()
|
||
|
||
async def janitor():
|
||
while True:
|
||
await asyncio.sleep(60)
|
||
now = time.time()
|
||
async with LOCK:
|
||
for sig, crawler in list(POOL.items()):
|
||
if now - LAST_USED[sig] > IDLE_TTL:
|
||
with suppress(Exception): await crawler.close()
|
||
POOL.pop(sig, None); LAST_USED.pop(sig, None)
|