mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-09-22 22:53:41 +00:00
13510aa753
* Files moved, imports all broken * Fix most imports and docstrings into * Fix the paths to the modules in the API docs * Add latest docstring and tutorial changes * Add a few pipelines that were lost in the inports * Fix a bunch of mypy warnings * Add latest docstring and tutorial changes * Create a file_classifier module * Add docs for file_classifier * Fixed most circular imports, now the REST API can start * Add latest docstring and tutorial changes * Tackling more mypy issues * Reintroduce from FARM and fix last mypy issues hopefully * Re-enable old-style imports * Fix some more import from the top-level package in an attempt to sort out circular imports * Fix some imports in tests to new-style to prevent failed class equalities from breaking tests * Change document_store into document_stores * Update imports in tutorials * Add latest docstring and tutorial changes * Probably fixes summarizer tests * Improve the old-style import allowing module imports (should work) * Try to fix the docs * Remove dedicated KnowledgeGraph page from autodocs * Remove dedicated GraphRetriever page from autodocs * Fix generate_docstrings.sh with an updated list of yaml files to look for * Fix some more modules in the docs * Fix the document stores docs too * Fix a small issue on Tutorial14 * Add latest docstring and tutorial changes * Add deprecation warning to old-style imports * Remove stray folder and import Dict into dense.py * Change import path for MLFlowLogger * Add old loggers path to the import path aliases * Fix debug output of convert_ipynb.py * Fix circular import on BaseRetriever * Missed one merge block * re-run tutorial 5 * Fix imports in tutorial 5 * Re-enable squad_to_dpr CLI from the root package and move get_batches_from_generator into document_stores.base * Add latest docstring and tutorial changes * Fix typo in utils __init__ * Fix a few more imports * Fix benchmarks too * New-style imports in test_knowledge_graph * Rollback setup.py * Rollback squad_to_dpr too Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
30 lines
820 B
Python
30 lines
820 B
Python
import re
|
|
|
|
|
|
def clean_wiki_text(text: str) -> str:
|
|
"""
|
|
Clean wikipedia text by removing multiple new lines, removing extremely short lines,
|
|
adding paragraph breaks and removing empty paragraphs
|
|
"""
|
|
# get rid of multiple new lines
|
|
while "\n\n" in text:
|
|
text = text.replace("\n\n", "\n")
|
|
|
|
# remove extremely short lines
|
|
lines = text.split("\n")
|
|
cleaned = []
|
|
for l in lines:
|
|
if len(l) > 30:
|
|
cleaned.append(l)
|
|
elif l[:2] == "==" and l[-2:] == "==":
|
|
cleaned.append(l)
|
|
text = "\n".join(cleaned)
|
|
|
|
# add paragraphs (identified by wiki section title which is always in format "==Some Title==")
|
|
text = text.replace("\n==", "\n\n\n==")
|
|
|
|
# remove empty paragrahps
|
|
text = re.sub(r"(==.*==\n\n\n)", "", text)
|
|
|
|
return text
|