yujunjun/llama-hub
1
0
Fork 0
mirror of https://github.com/run-llama/llama-hub.git synced 2025-11-11 07:14:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Added an option to choose the language in Wikipedia texts (#142)

Browse Source
This commit is contained in:
Emmanuel 2023-03-27 22:56:54 -03:00 committed by GitHub
parent d0185fc543
commit e43264cbbc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
loader_hub/wikipedia/base.py
View File

@ -12,17 +12,18 @@ class WikipediaReader(BaseReader):
""" """
def load_data(self, pages: List[str], **load_kwargs: Any) -> List[Document]: def load_data(self, pages: List[str], lang: str = "en", **load_kwargs: Any) -> List[Document]:
"""Load data from the input directory. """Load data from the input directory.
Args: Args:
pages (List[str]): List of pages to read. pages (List[str]): List of pages to read.
lang (str): language of wikipedia texts (default English)
""" """
import wikipedia import wikipedia
results = [] results = []
for page in pages: for page in pages:
wikipedia.set_lang(lang)
page_content = wikipedia.page(page, **load_kwargs).content page_content = wikipedia.page(page, **load_kwargs).content
results.append(Document(page_content)) results.append(Document(page_content))
return results return results