2024-11-18 21:00:06 +08:00
|
|
|
import os, sys
|
|
|
|
|
# append the parent directory to the sys.path
|
|
|
|
|
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
|
sys.path.append(parent_dir)
|
|
|
|
|
parent_parent_dir = os.path.dirname(parent_dir)
|
|
|
|
|
sys.path.append(parent_parent_dir)
|
|
|
|
|
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
|
|
|
|
|
__data__ = os.path.join(__location__, "__data")
|
2024-11-15 20:16:13 +08:00
|
|
|
import asyncio
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
import aiohttp
|
|
|
|
|
import json
|
2024-11-18 21:00:06 +08:00
|
|
|
from crawl4ai import AsyncWebCrawler, CacheMode
|
2024-11-15 20:16:13 +08:00
|
|
|
from crawl4ai.content_filter_strategy import BM25ContentFilter
|
|
|
|
|
|
|
|
|
|
# 1. File Download Processing Example
|
|
|
|
|
async def download_example():
|
|
|
|
|
"""Example of downloading files from Python.org"""
|
|
|
|
|
# downloads_path = os.path.join(os.getcwd(), "downloads")
|
|
|
|
|
downloads_path = os.path.join(Path.home(), ".crawl4ai", "downloads")
|
|
|
|
|
os.makedirs(downloads_path, exist_ok=True)
|
|
|
|
|
|
|
|
|
|
print(f"Downloads will be saved to: {downloads_path}")
|
|
|
|
|
|
|
|
|
|
async with AsyncWebCrawler(
|
|
|
|
|
accept_downloads=True,
|
|
|
|
|
downloads_path=downloads_path,
|
|
|
|
|
verbose=True
|
|
|
|
|
) as crawler:
|
|
|
|
|
result = await crawler.arun(
|
|
|
|
|
url="https://www.python.org/downloads/",
|
|
|
|
|
js_code="""
|
|
|
|
|
// Find and click the first Windows installer link
|
|
|
|
|
const downloadLink = document.querySelector('a[href$=".exe"]');
|
|
|
|
|
if (downloadLink) {
|
|
|
|
|
console.log('Found download link:', downloadLink.href);
|
|
|
|
|
downloadLink.click();
|
|
|
|
|
} else {
|
|
|
|
|
console.log('No .exe download link found');
|
|
|
|
|
}
|
|
|
|
|
""",
|
2024-11-18 21:00:06 +08:00
|
|
|
delay_before_return_html=1, # Wait 5 seconds to ensure download starts
|
|
|
|
|
cache_mode=CacheMode.BYPASS
|
2024-11-15 20:16:13 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
if result.downloaded_files:
|
|
|
|
|
print("\nDownload successful!")
|
|
|
|
|
print("Downloaded files:")
|
|
|
|
|
for file_path in result.downloaded_files:
|
|
|
|
|
print(f"- {file_path}")
|
|
|
|
|
print(f" File size: {os.path.getsize(file_path) / (1024*1024):.2f} MB")
|
|
|
|
|
else:
|
|
|
|
|
print("\nNo files were downloaded")
|
|
|
|
|
|
|
|
|
|
# 2. Content Filtering with BM25 Example
|
|
|
|
|
async def content_filtering_example():
|
|
|
|
|
"""Example of using the new BM25 content filtering"""
|
|
|
|
|
async with AsyncWebCrawler(verbose=True) as crawler:
|
|
|
|
|
# Create filter with custom query for OpenAI's blog
|
|
|
|
|
content_filter = BM25ContentFilter(
|
2024-11-18 21:00:06 +08:00
|
|
|
# user_query="Investment and fundraising",
|
|
|
|
|
# user_query="Robotic",
|
2024-11-15 20:16:13 +08:00
|
|
|
bm25_threshold=1.0
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
result = await crawler.arun(
|
2024-11-18 21:00:06 +08:00
|
|
|
url="https://techcrunch.com/",
|
|
|
|
|
content_filter=content_filter,
|
|
|
|
|
cache_mode=CacheMode.BYPASS
|
2024-11-15 20:16:13 +08:00
|
|
|
)
|
|
|
|
|
|
2024-11-18 21:00:06 +08:00
|
|
|
print(f"Filtered content: {len(result.fit_markdown)}")
|
|
|
|
|
print(f"Filtered content: {result.fit_markdown}")
|
|
|
|
|
|
|
|
|
|
# Save html
|
|
|
|
|
with open(os.path.join(__data__, "techcrunch.html"), "w") as f:
|
|
|
|
|
f.write(result.fit_html)
|
|
|
|
|
|
|
|
|
|
with open(os.path.join(__data__, "filtered_content.md"), "w") as f:
|
|
|
|
|
f.write(result.fit_markdown)
|
2024-11-15 20:16:13 +08:00
|
|
|
|
|
|
|
|
# 3. Local File and Raw HTML Processing Example
|
|
|
|
|
async def local_and_raw_html_example():
|
|
|
|
|
"""Example of processing local files and raw HTML"""
|
|
|
|
|
# Create a sample HTML file
|
2024-11-18 21:00:06 +08:00
|
|
|
sample_file = os.path.join(__data__, "sample.html")
|
2024-11-15 20:16:13 +08:00
|
|
|
with open(sample_file, "w") as f:
|
|
|
|
|
f.write("""
|
|
|
|
|
<html><body>
|
|
|
|
|
<h1>Test Content</h1>
|
|
|
|
|
<p>This is a test paragraph.</p>
|
|
|
|
|
</body></html>
|
|
|
|
|
""")
|
|
|
|
|
|
|
|
|
|
async with AsyncWebCrawler(verbose=True) as crawler:
|
|
|
|
|
# Process local file
|
|
|
|
|
local_result = await crawler.arun(
|
|
|
|
|
url=f"file://{os.path.abspath(sample_file)}"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Process raw HTML
|
|
|
|
|
raw_html = """
|
|
|
|
|
<html><body>
|
|
|
|
|
<h1>Raw HTML Test</h1>
|
|
|
|
|
<p>This is a test of raw HTML processing.</p>
|
|
|
|
|
</body></html>
|
|
|
|
|
"""
|
|
|
|
|
raw_result = await crawler.arun(
|
|
|
|
|
url=f"raw:{raw_html}"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Clean up
|
|
|
|
|
os.remove(sample_file)
|
|
|
|
|
|
|
|
|
|
print("Local file content:", local_result.markdown)
|
|
|
|
|
print("\nRaw HTML content:", raw_result.markdown)
|
|
|
|
|
|
|
|
|
|
# 4. Browser Management Example
|
|
|
|
|
async def browser_management_example():
|
|
|
|
|
"""Example of using enhanced browser management features"""
|
|
|
|
|
# Use the specified user directory path
|
|
|
|
|
user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile")
|
|
|
|
|
os.makedirs(user_data_dir, exist_ok=True)
|
|
|
|
|
|
|
|
|
|
print(f"Browser profile will be saved to: {user_data_dir}")
|
|
|
|
|
|
|
|
|
|
async with AsyncWebCrawler(
|
|
|
|
|
use_managed_browser=True,
|
|
|
|
|
user_data_dir=user_data_dir,
|
|
|
|
|
headless=False,
|
|
|
|
|
verbose=True
|
|
|
|
|
) as crawler:
|
2024-11-18 21:00:06 +08:00
|
|
|
|
|
|
|
|
result = await crawler.arun(
|
|
|
|
|
url="https://crawl4ai.com",
|
|
|
|
|
# session_id="persistent_session_1",
|
|
|
|
|
cache_mode=CacheMode.BYPASS
|
|
|
|
|
)
|
2024-11-15 20:16:13 +08:00
|
|
|
# Use GitHub as an example - it's a good test for browser management
|
|
|
|
|
# because it requires proper browser handling
|
|
|
|
|
result = await crawler.arun(
|
|
|
|
|
url="https://github.com/trending",
|
2024-11-18 21:00:06 +08:00
|
|
|
# session_id="persistent_session_1",
|
|
|
|
|
cache_mode=CacheMode.BYPASS
|
2024-11-15 20:16:13 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
print("\nBrowser session result:", result.success)
|
|
|
|
|
if result.success:
|
|
|
|
|
print("Page title:", result.metadata.get('title', 'No title found'))
|
|
|
|
|
|
|
|
|
|
# 5. API Usage Example
|
|
|
|
|
async def api_example():
|
|
|
|
|
"""Example of using the new API endpoints"""
|
2024-11-18 21:00:06 +08:00
|
|
|
api_token = os.getenv('CRAWL4AI_API_TOKEN') or "test_api_code"
|
|
|
|
|
headers = {'Authorization': f'Bearer {api_token}'}
|
2024-11-15 20:16:13 +08:00
|
|
|
async with aiohttp.ClientSession() as session:
|
|
|
|
|
# Submit crawl job
|
|
|
|
|
crawl_request = {
|
|
|
|
|
"urls": ["https://news.ycombinator.com"], # Hacker News as an example
|
|
|
|
|
"extraction_config": {
|
|
|
|
|
"type": "json_css",
|
|
|
|
|
"params": {
|
2024-11-18 21:00:06 +08:00
|
|
|
"schema": {
|
|
|
|
|
"name": "Hacker News Articles",
|
|
|
|
|
"baseSelector": ".athing",
|
|
|
|
|
"fields": [
|
|
|
|
|
{
|
|
|
|
|
"name": "title",
|
|
|
|
|
"selector": ".title a",
|
|
|
|
|
"type": "text"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"name": "score",
|
|
|
|
|
"selector": ".score",
|
|
|
|
|
"type": "text"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"name": "url",
|
|
|
|
|
"selector": ".title a",
|
|
|
|
|
"type": "attribute",
|
|
|
|
|
"attribute": "href"
|
|
|
|
|
}
|
|
|
|
|
]
|
2024-11-15 20:16:13 +08:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"crawler_params": {
|
|
|
|
|
"headless": True,
|
2024-11-18 21:00:06 +08:00
|
|
|
# "use_managed_browser": True
|
2024-11-15 20:16:13 +08:00
|
|
|
},
|
2024-11-18 21:00:06 +08:00
|
|
|
"cache_mode": "bypass",
|
|
|
|
|
# "screenshot": True,
|
|
|
|
|
# "magic": True
|
2024-11-15 20:16:13 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async with session.post(
|
|
|
|
|
"http://localhost:11235/crawl",
|
2024-11-18 21:00:06 +08:00
|
|
|
json=crawl_request,
|
|
|
|
|
headers=headers
|
2024-11-15 20:16:13 +08:00
|
|
|
) as response:
|
|
|
|
|
task_data = await response.json()
|
|
|
|
|
task_id = task_data["task_id"]
|
|
|
|
|
|
|
|
|
|
# Check task status
|
2024-11-18 21:00:06 +08:00
|
|
|
while True:
|
|
|
|
|
async with session.get(
|
|
|
|
|
f"http://localhost:11235/task/{task_id}",
|
|
|
|
|
headers=headers
|
|
|
|
|
) as status_response:
|
|
|
|
|
result = await status_response.json()
|
|
|
|
|
print(f"Task result: {result}")
|
|
|
|
|
|
|
|
|
|
if result["status"] == "completed":
|
|
|
|
|
break
|
|
|
|
|
else:
|
|
|
|
|
await asyncio.sleep(1)
|
2024-11-15 20:16:13 +08:00
|
|
|
|
|
|
|
|
# Main execution
|
|
|
|
|
async def main():
|
2024-11-18 21:00:06 +08:00
|
|
|
# print("Running Crawl4AI feature examples...")
|
2024-11-15 20:16:13 +08:00
|
|
|
|
2024-11-18 21:00:06 +08:00
|
|
|
# print("\n1. Running Download Example:")
|
2024-11-15 20:16:13 +08:00
|
|
|
await download_example()
|
|
|
|
|
|
2024-11-18 21:00:06 +08:00
|
|
|
# print("\n2. Running Content Filtering Example:")
|
2024-11-15 20:16:13 +08:00
|
|
|
await content_filtering_example()
|
|
|
|
|
|
2024-11-18 21:00:06 +08:00
|
|
|
# print("\n3. Running Local and Raw HTML Example:")
|
2024-11-15 20:16:13 +08:00
|
|
|
await local_and_raw_html_example()
|
|
|
|
|
|
2024-11-18 21:00:06 +08:00
|
|
|
# print("\n4. Running Browser Management Example:")
|
2024-11-15 20:16:13 +08:00
|
|
|
await browser_management_example()
|
|
|
|
|
|
2024-11-18 21:00:06 +08:00
|
|
|
# print("\n5. Running API Example:")
|
2024-11-15 20:16:13 +08:00
|
|
|
await api_example()
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
asyncio.run(main())
|
