loaders:
  - type: loaders.CustomPythonLoader
    search_path: [../../../haystack/nodes/connector]
    modules: ["crawler"]
    ignore_when_discovered: ["__init__"]
processors:
  - type: filter
    expression:
    documented_only: true
    do_not_filter_modules: false
    skip_empty_modules: true
  - type: smart
  - type: crossref
renderer:
  type: renderers.ReadmeRenderer
  excerpt: The Crawler scrapes the text from a website, creates a Haystack Document object out of it, and saves it to a JSON file.
  category_slug: haystack-classes
  title: Crawler API
  slug: crawler-api
  order: 10
  markdown:
    descriptive_class_title: false
    descriptive_module_title: true
    add_method_class_prefix: true
    add_member_class_prefix: false
    filename: crawler_api.md