yujunjun/haystack
1
0
Fork 0
mirror of https://github.com/deepset-ai/haystack.git synced 2025-10-20 04:18:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix download ntlk preprocessor (#852)

Browse Source
This commit is contained in:
Tu NGUYEN 2021-02-21 10:17:50 +01:00 committed by GitHub
parent e641bff7a6
commit ba91a90dd6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
haystack/preprocessor/preprocessor.py
View File

@ -44,7 +44,11 @@ class PreProcessor(BasePreProcessor):
to True, the individual split will always have complete sentences & to True, the individual split will always have complete sentences &
the number of words will be <= split_length. the number of words will be <= split_length.
""" """
nltk.download("punkt") try:
nltk.data.find('tokenizers/punkt')
except LookupError:
nltk.download('punkt')
self.clean_whitespace = clean_whitespace self.clean_whitespace = clean_whitespace
self.clean_header_footer = clean_header_footer self.clean_header_footer = clean_header_footer
self.clean_empty_lines = clean_empty_lines self.clean_empty_lines = clean_empty_lines