mirror of
https://github.com/datahub-project/datahub.git
synced 2025-12-16 12:38:13 +00:00
docs: revert versioned docs (#15486)
This commit is contained in:
parent
ef46ed1d7a
commit
0a14181fea
2
.gitignore
vendored
2
.gitignore
vendored
@ -74,8 +74,6 @@ metadata-ingestion/generated/**
|
|||||||
|
|
||||||
# docs
|
# docs
|
||||||
docs/generated/
|
docs/generated/
|
||||||
docs-website/versioned_docs/
|
|
||||||
docs-website/versioned_sidebars/
|
|
||||||
tmp*
|
tmp*
|
||||||
temp/**
|
temp/**
|
||||||
|
|
||||||
|
|||||||
@ -119,13 +119,7 @@ task yarnGenerate(type: YarnTask, dependsOn: [yarnInstall,
|
|||||||
args = ['run', 'generate']
|
args = ['run', 'generate']
|
||||||
}
|
}
|
||||||
|
|
||||||
task downloadHistoricalVersions(type: Exec, dependsOn: installPythonDeps) {
|
task yarnStart(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
||||||
workingDir '.'
|
|
||||||
commandLine 'bash', '-c',
|
|
||||||
venv_activate_command + 'python download_historical_versions.py'
|
|
||||||
}
|
|
||||||
|
|
||||||
task yarnStart(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate, downloadHistoricalVersions]) {
|
|
||||||
args = ['run', 'start']
|
args = ['run', 'start']
|
||||||
}
|
}
|
||||||
task fastReload(type: YarnTask) {
|
task fastReload(type: YarnTask) {
|
||||||
@ -147,7 +141,7 @@ task serve(type: YarnTask, dependsOn: [yarnInstall] ) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
task yarnBuild(type: YarnTask, dependsOn: [yarnLint, yarnGenerate, downloadHistoricalVersions]) {
|
task yarnBuild(type: YarnTask, dependsOn: [yarnLint, yarnGenerate]) {
|
||||||
inputs.files(projectMdFiles)
|
inputs.files(projectMdFiles)
|
||||||
inputs.file("package.json").withPathSensitivity(PathSensitivity.RELATIVE)
|
inputs.file("package.json").withPathSensitivity(PathSensitivity.RELATIVE)
|
||||||
inputs.dir("src").withPathSensitivity(PathSensitivity.RELATIVE)
|
inputs.dir("src").withPathSensitivity(PathSensitivity.RELATIVE)
|
||||||
@ -178,7 +172,6 @@ clean {
|
|||||||
delete 'just'
|
delete 'just'
|
||||||
delete 'sphinx/venv'
|
delete 'sphinx/venv'
|
||||||
delete 'sphinx/_build'
|
delete 'sphinx/_build'
|
||||||
delete 'versioned_docs'
|
|
||||||
delete venv_name
|
delete venv_name
|
||||||
delete fileTree(dir: 'genDocs', exclude: '.gitignore')
|
delete fileTree(dir: 'genDocs', exclude: '.gitignore')
|
||||||
delete fileTree(dir: 'docs', exclude: '.gitignore')
|
delete fileTree(dir: 'docs', exclude: '.gitignore')
|
||||||
|
|||||||
@ -338,7 +338,7 @@ module.exports = {
|
|||||||
lastVersion: "current",
|
lastVersion: "current",
|
||||||
versions: {
|
versions: {
|
||||||
current: {
|
current: {
|
||||||
label: "Next",
|
label: "1.3.0",
|
||||||
banner: 'none',
|
banner: 'none',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@ -1,99 +0,0 @@
|
|||||||
import json
|
|
||||||
import os
|
|
||||||
import tarfile
|
|
||||||
import time
|
|
||||||
import requests
|
|
||||||
import shutil
|
|
||||||
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
|
|
||||||
|
|
||||||
repo_url = "https://api.github.com/repos/datahub-project/static-assets"
|
|
||||||
|
|
||||||
|
|
||||||
def download_file(url, destination):
|
|
||||||
response = requests.get(url, stream=True)
|
|
||||||
response.raise_for_status()
|
|
||||||
with open(destination, "wb") as f:
|
|
||||||
for chunk in response.iter_content(chunk_size=8192):
|
|
||||||
f.write(chunk)
|
|
||||||
|
|
||||||
|
|
||||||
@retry(
|
|
||||||
stop=stop_after_attempt(10),
|
|
||||||
wait=wait_exponential(multiplier=1, min=1, max=30),
|
|
||||||
retry=retry_if_exception_type(Exception)
|
|
||||||
)
|
|
||||||
def fetch_urls(
|
|
||||||
repo_url: str, folder_path: str, file_format: str, active_versions: list
|
|
||||||
):
|
|
||||||
api_url = f"{repo_url}/contents/{folder_path}"
|
|
||||||
response = requests.get(api_url)
|
|
||||||
if response.status_code == 403 or (500 <= response.status_code < 600):
|
|
||||||
raise Exception(f"HTTP Error {response.status_code}: {response.reason}")
|
|
||||||
response.raise_for_status()
|
|
||||||
data = response.json()
|
|
||||||
urls = [
|
|
||||||
file["download_url"]
|
|
||||||
for file in data
|
|
||||||
if file["name"].endswith(file_format) and any(version in file["name"] for version in active_versions)
|
|
||||||
]
|
|
||||||
print(urls)
|
|
||||||
return urls
|
|
||||||
|
|
||||||
|
|
||||||
def extract_tar_file(destination_path):
|
|
||||||
with tarfile.open(destination_path, "r:gz") as tar:
|
|
||||||
tar.extractall()
|
|
||||||
os.remove(destination_path)
|
|
||||||
|
|
||||||
def get_active_versions():
|
|
||||||
# read versions.json
|
|
||||||
with open("versions.json") as f:
|
|
||||||
versions = json.load(f)
|
|
||||||
return versions
|
|
||||||
|
|
||||||
def clear_directory(directory):
|
|
||||||
if os.path.exists(directory):
|
|
||||||
shutil.rmtree(directory)
|
|
||||||
os.makedirs(directory)
|
|
||||||
|
|
||||||
def download_versioned_docs(folder_path: str, destination_dir: str, file_format: str):
|
|
||||||
clear_directory(destination_dir) # Clear the directory before downloading
|
|
||||||
|
|
||||||
active_versions = get_active_versions()
|
|
||||||
urls = fetch_urls(repo_url, folder_path, file_format, active_versions)
|
|
||||||
|
|
||||||
for url in urls:
|
|
||||||
filename = os.path.basename(url)
|
|
||||||
destination_path = os.path.join(destination_dir, filename)
|
|
||||||
|
|
||||||
version = ".".join(filename.split(".")[:3])
|
|
||||||
extracted_path = os.path.join(destination_dir, version)
|
|
||||||
print("extracted_path", extracted_path)
|
|
||||||
if os.path.exists(extracted_path):
|
|
||||||
print(f"{extracted_path} already exists, skipping downloads")
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
download_file(url, destination_path)
|
|
||||||
print(f"Downloaded {filename} to {destination_dir}")
|
|
||||||
if file_format == ".tar.gz":
|
|
||||||
extract_tar_file(destination_path)
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Error while downloading {filename}: {e}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
download_versioned_docs(
|
|
||||||
folder_path="versioned_docs",
|
|
||||||
destination_dir="versioned_docs",
|
|
||||||
file_format=".tar.gz",
|
|
||||||
)
|
|
||||||
download_versioned_docs(
|
|
||||||
folder_path="versioned_sidebars",
|
|
||||||
destination_dir="versioned_sidebars",
|
|
||||||
file_format=".json",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -1,3 +0,0 @@
|
|||||||
[
|
|
||||||
"1.3.0"
|
|
||||||
]
|
|
||||||
Loading…
x
Reference in New Issue
Block a user