mirror of
https://github.com/datahub-project/datahub.git
synced 2025-12-15 20:16:52 +00:00
docs: revert versioned docs (#15486)
This commit is contained in:
parent
ef46ed1d7a
commit
0a14181fea
2
.gitignore
vendored
2
.gitignore
vendored
@ -74,8 +74,6 @@ metadata-ingestion/generated/**
|
||||
|
||||
# docs
|
||||
docs/generated/
|
||||
docs-website/versioned_docs/
|
||||
docs-website/versioned_sidebars/
|
||||
tmp*
|
||||
temp/**
|
||||
|
||||
|
||||
@ -119,13 +119,7 @@ task yarnGenerate(type: YarnTask, dependsOn: [yarnInstall,
|
||||
args = ['run', 'generate']
|
||||
}
|
||||
|
||||
task downloadHistoricalVersions(type: Exec, dependsOn: installPythonDeps) {
|
||||
workingDir '.'
|
||||
commandLine 'bash', '-c',
|
||||
venv_activate_command + 'python download_historical_versions.py'
|
||||
}
|
||||
|
||||
task yarnStart(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate, downloadHistoricalVersions]) {
|
||||
task yarnStart(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
||||
args = ['run', 'start']
|
||||
}
|
||||
task fastReload(type: YarnTask) {
|
||||
@ -147,7 +141,7 @@ task serve(type: YarnTask, dependsOn: [yarnInstall] ) {
|
||||
}
|
||||
|
||||
|
||||
task yarnBuild(type: YarnTask, dependsOn: [yarnLint, yarnGenerate, downloadHistoricalVersions]) {
|
||||
task yarnBuild(type: YarnTask, dependsOn: [yarnLint, yarnGenerate]) {
|
||||
inputs.files(projectMdFiles)
|
||||
inputs.file("package.json").withPathSensitivity(PathSensitivity.RELATIVE)
|
||||
inputs.dir("src").withPathSensitivity(PathSensitivity.RELATIVE)
|
||||
@ -178,7 +172,6 @@ clean {
|
||||
delete 'just'
|
||||
delete 'sphinx/venv'
|
||||
delete 'sphinx/_build'
|
||||
delete 'versioned_docs'
|
||||
delete venv_name
|
||||
delete fileTree(dir: 'genDocs', exclude: '.gitignore')
|
||||
delete fileTree(dir: 'docs', exclude: '.gitignore')
|
||||
|
||||
@ -338,7 +338,7 @@ module.exports = {
|
||||
lastVersion: "current",
|
||||
versions: {
|
||||
current: {
|
||||
label: "Next",
|
||||
label: "1.3.0",
|
||||
banner: 'none',
|
||||
},
|
||||
},
|
||||
|
||||
@ -1,99 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import tarfile
|
||||
import time
|
||||
import requests
|
||||
import shutil
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
|
||||
|
||||
repo_url = "https://api.github.com/repos/datahub-project/static-assets"
|
||||
|
||||
|
||||
def download_file(url, destination):
|
||||
response = requests.get(url, stream=True)
|
||||
response.raise_for_status()
|
||||
with open(destination, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(10),
|
||||
wait=wait_exponential(multiplier=1, min=1, max=30),
|
||||
retry=retry_if_exception_type(Exception)
|
||||
)
|
||||
def fetch_urls(
|
||||
repo_url: str, folder_path: str, file_format: str, active_versions: list
|
||||
):
|
||||
api_url = f"{repo_url}/contents/{folder_path}"
|
||||
response = requests.get(api_url)
|
||||
if response.status_code == 403 or (500 <= response.status_code < 600):
|
||||
raise Exception(f"HTTP Error {response.status_code}: {response.reason}")
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
urls = [
|
||||
file["download_url"]
|
||||
for file in data
|
||||
if file["name"].endswith(file_format) and any(version in file["name"] for version in active_versions)
|
||||
]
|
||||
print(urls)
|
||||
return urls
|
||||
|
||||
|
||||
def extract_tar_file(destination_path):
|
||||
with tarfile.open(destination_path, "r:gz") as tar:
|
||||
tar.extractall()
|
||||
os.remove(destination_path)
|
||||
|
||||
def get_active_versions():
|
||||
# read versions.json
|
||||
with open("versions.json") as f:
|
||||
versions = json.load(f)
|
||||
return versions
|
||||
|
||||
def clear_directory(directory):
|
||||
if os.path.exists(directory):
|
||||
shutil.rmtree(directory)
|
||||
os.makedirs(directory)
|
||||
|
||||
def download_versioned_docs(folder_path: str, destination_dir: str, file_format: str):
|
||||
clear_directory(destination_dir) # Clear the directory before downloading
|
||||
|
||||
active_versions = get_active_versions()
|
||||
urls = fetch_urls(repo_url, folder_path, file_format, active_versions)
|
||||
|
||||
for url in urls:
|
||||
filename = os.path.basename(url)
|
||||
destination_path = os.path.join(destination_dir, filename)
|
||||
|
||||
version = ".".join(filename.split(".")[:3])
|
||||
extracted_path = os.path.join(destination_dir, version)
|
||||
print("extracted_path", extracted_path)
|
||||
if os.path.exists(extracted_path):
|
||||
print(f"{extracted_path} already exists, skipping downloads")
|
||||
continue
|
||||
try:
|
||||
download_file(url, destination_path)
|
||||
print(f"Downloaded {filename} to {destination_dir}")
|
||||
if file_format == ".tar.gz":
|
||||
extract_tar_file(destination_path)
|
||||
except Exception as e:
|
||||
print(f"Error while downloading {filename}: {e}")
|
||||
continue
|
||||
|
||||
|
||||
def main():
|
||||
download_versioned_docs(
|
||||
folder_path="versioned_docs",
|
||||
destination_dir="versioned_docs",
|
||||
file_format=".tar.gz",
|
||||
)
|
||||
download_versioned_docs(
|
||||
folder_path="versioned_sidebars",
|
||||
destination_dir="versioned_sidebars",
|
||||
file_format=".json",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,3 +0,0 @@
|
||||
[
|
||||
"1.3.0"
|
||||
]
|
||||
Loading…
x
Reference in New Issue
Block a user