diff --git a/MANIFEST.in b/MANIFEST.in index 540b720..73a0e00 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ -include requirements.txt \ No newline at end of file +include requirements.txt +recursive-include crawl4ai/js_snippet *.js \ No newline at end of file diff --git a/docs/examples/quickstart_async.config.py b/docs/examples/quickstart_async.config.py index e1feba5..ff31268 100644 --- a/docs/examples/quickstart_async.config.py +++ b/docs/examples/quickstart_async.config.py @@ -142,6 +142,7 @@ async def extract_structured_data_using_llm(provider: str, api_token: str = None crawler_config = CrawlerRunConfig( cache_mode=CacheMode.BYPASS, word_count_threshold=1, + page_timeout = 80000, extraction_strategy=LLMExtractionStrategy( provider=provider, api_token=api_token, @@ -497,21 +498,21 @@ async def main(): # Advanced examples # await extract_structured_data_using_css_extractor() - # await extract_structured_data_using_llm("openai/gpt-4o", os.getenv("OPENAI_API_KEY")) + await extract_structured_data_using_llm("openai/gpt-4o", os.getenv("OPENAI_API_KEY")) # await crawl_dynamic_content_pages_method_1() # await crawl_dynamic_content_pages_method_2() # Browser comparisons - await crawl_custom_browser_type() + # await crawl_custom_browser_type() # Performance testing # await speed_comparison() # Screenshot example - await capture_and_save_screenshot( - "https://www.example.com", - os.path.join(__location__, "tmp/example_screenshot.jpg") - ) + # await capture_and_save_screenshot( + # "https://www.example.com", + # os.path.join(__location__, "tmp/example_screenshot.jpg") + # ) if __name__ == "__main__": asyncio.run(main()) \ No newline at end of file diff --git a/setup.py b/setup.py index e6840cd..bc6b00c 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,9 @@ setup( author_email="unclecode@kidocode.com", license="MIT", packages=find_packages(), + package_data={ + 'crawl4ai': ['js_snippet/*.js'] # This matches the exact path structure + }, install_requires=default_requirements + ["playwright", "aiofiles"], # Added aiofiles extras_require={