loaders:
  - type: python
    search_path: [../../../../haystack/nodes/connector]
    modules: ['crawler']
    ignore_when_discovered: ['__init__']
processors:
  - type: filter
    expression: 
    documented_only: true
    do_not_filter_modules: false
    skip_empty_modules: true
  - type: smart
  - type: crossref
renderer:
   type: renderers.ReadmeRenderer
   excerpt: The Crawler scrapes the text from a website, creates a Haystack Document object out of it, and saves it to a JSON file.
   category: 6310ca73c622850ddd3875a2
   title: Crawler API
   slug: crawler-api
   order: 10
   markdown:
     descriptive_class_title: false
     descriptive_module_title: true
     add_method_class_prefix: true
     add_member_class_prefix: false
     filename: crawler_api.md