yujunjun/haystack
1
0
Fork 0
mirror of https://github.com/deepset-ai/haystack.git synced 2025-07-26 02:10:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix crawler (#2591)

Browse Source
This commit is contained in:
bogdankostic 2022-05-24 12:34:31 +02:00 committed by GitHub
parent 867695ad0c
commit 1ab2b977c0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
haystack/nodes/connector/crawler.py
View File

@ -280,7 +280,7 @@ class Crawler(BaseComponent):
self, base_url: str, filter_urls: Optional[List] = None, existed_links: List = None self, base_url: str, filter_urls: Optional[List] = None, existed_links: List = None
) -> set: ) -> set:
self.driver.get(base_url) self.driver.get(base_url)
a_elements = self.driver.find_elements_by_tag_name("a") a_elements = self.driver.find_elements_by_xpath("//a[@href]")
sub_links = set() sub_links = set()
if not (existed_links and base_url in existed_links): if not (existed_links and base_url in existed_links):
if filter_urls: if filter_urls: