From e43264cbbcf03721b1cafbc8baf9d64c710dffbd Mon Sep 17 00:00:00 2001 From: Emmanuel Date: Mon, 27 Mar 2023 22:56:54 -0300 Subject: [PATCH] feat: Added an option to choose the language in Wikipedia texts (#142) --- loader_hub/wikipedia/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/loader_hub/wikipedia/base.py b/loader_hub/wikipedia/base.py index f60edcd8..63c2c232 100644 --- a/loader_hub/wikipedia/base.py +++ b/loader_hub/wikipedia/base.py @@ -12,17 +12,18 @@ class WikipediaReader(BaseReader): """ - def load_data(self, pages: List[str], **load_kwargs: Any) -> List[Document]: + def load_data(self, pages: List[str], lang: str = "en", **load_kwargs: Any) -> List[Document]: """Load data from the input directory. Args: pages (List[str]): List of pages to read. - + lang (str): language of wikipedia texts (default English) """ import wikipedia results = [] for page in pages: + wikipedia.set_lang(lang) page_content = wikipedia.page(page, **load_kwargs).content results.append(Document(page_content)) return results