yujunjun/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-06 20:07:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refa: Remove HuggingFace repo downloads (#11048)

Browse Source 
### What problem does this PR solve?

- Removed download_model function and HuggingFace repo download loop

### Type of change

- [x] Refactoring
This commit is contained in:
Liu An 2025-11-06 11:53:33 +08:00 committed by GitHub
parent adbb8319e0
commit 15c75bbf15
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 12 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
download_deps.py
View File

@ -9,12 +9,13 @@
# ]
# ///
from huggingface_hub import snapshot_download
from typing import Union
import nltk
import argparse
import os
import urllib.request
import argparse
from typing import Union
import nltk
def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]:
if use_china_mirrors:
@ -38,25 +39,21 @@ def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]:
["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chromedriver-linux64.zip", "chromedriver-linux64-121-0-6167-85"],
]
repos = [
"InfiniFlow/text_concat_xgb_v1.0",
"InfiniFlow/deepdoc",
"InfiniFlow/huqie",
]
def download_model(repo_id):
local_dir = os.path.abspath(os.path.join("huggingface.co", repo_id))
os.makedirs(local_dir, exist_ok=True)
snapshot_download(repo_id=repo_id, local_dir=local_dir)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Download dependencies with optional China mirror support')
parser.add_argument('--china-mirrors', action='store_true', help='Use China-accessible mirrors for downloads')
parser = argparse.ArgumentParser(description="Download dependencies with optional China mirror support")
parser.add_argument("--china-mirrors", action="store_true", help="Use China-accessible mirrors for downloads")
args = parser.parse_args()
urls = get_urls(args.china_mirrors)
for url in urls:
download_url = url[0] if isinstance(url, list) else url
filename = url[1] if isinstance(url, list) else url.split("/")[-1]
@ -64,11 +61,7 @@ if __name__ == "__main__":
if not os.path.exists(filename):
urllib.request.urlretrieve(download_url, filename)
local_dir = os.path.abspath('nltk_data')
for data in ['wordnet', 'punkt', 'punkt_tab']:
local_dir = os.path.abspath("nltk_data")
for data in ["wordnet", "punkt", "punkt_tab"]:
print(f"Downloading nltk {data}...")
nltk.download(data, download_dir=local_dir)
for repo_id in repos:
print(f"Downloading huggingface repo {repo_id}...")
download_model(repo_id)