yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-03 12:16:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: modify the archived version & update code to download only the a… (#11228)

Browse Source
This commit is contained in:
Hyejin Yoon 2024-08-23 12:28:07 +09:00 committed by GitHub
parent ef6a410091
commit e1a29080ff
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 16 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs-website/docusaurus.config.js
View File

@ -170,14 +170,6 @@ module.exports = {
value: '<div class="dropdown__link"><b>Archived versions</b></div>',
},
{
value: `
<a class="dropdown__link" href="https://docs-website-qou70o69f-acryldata.vercel.app/docs/features">0.14.0
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
</a>
`,
type: "html",
},
{
value: `
<a class="dropdown__link" href="https://docs-website-lzxh86531-acryldata.vercel.app/docs/features">0.13.0
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>

21
docs-website/download_historical_versions.py
View File

@ -3,6 +3,7 @@ import os
import tarfile
import time
import urllib.request
import shutil
repo_url = "https://api.github.com/repos/datahub-project/static-assets"
@ -18,7 +19,7 @@ def download_file(url, destination):
def fetch_urls(
repo_url: str, folder_path: str, file_format: str, max_retries=3, retry_delay=5
repo_url: str, folder_path: str, file_format: str, active_versions: list, max_retries=3, retry_delay=5
):
api_url = f"{repo_url}/contents/{folder_path}"
for attempt in range(max_retries + 1):
@ -30,7 +31,7 @@ def fetch_urls(
urls = [
file["download_url"]
for file in json.loads(data)
if file["name"].endswith(file_format)
if file["name"].endswith(file_format) and any(version in file["name"] for version in active_versions)
]
print(urls)
return urls
@ -48,12 +49,22 @@ def extract_tar_file(destination_path):
tar.extractall()
os.remove(destination_path)
def get_active_versions():
# read versions.json
with open("versions.json") as f:
versions = json.load(f)
return versions
def clear_directory(directory):
if os.path.exists(directory):
shutil.rmtree(directory)
os.makedirs(directory)
def download_versioned_docs(folder_path: str, destination_dir: str, file_format: str):
if not os.path.exists(destination_dir):
os.makedirs(destination_dir)
clear_directory(destination_dir) # Clear the directory before downloading
urls = fetch_urls(repo_url, folder_path, file_format)
active_versions = get_active_versions()
urls = fetch_urls(repo_url, folder_path, file_format, active_versions)
for url in urls:
filename = os.path.basename(url)