mirror of
https://github.com/datahub-project/datahub.git
synced 2025-12-29 19:07:33 +00:00
feat(docs): move versioned_sidebars to static-assets (#8743)
This commit is contained in:
parent
97a77d8d0b
commit
bebee88100
1
.gitignore
vendored
1
.gitignore
vendored
@ -70,6 +70,7 @@ metadata-ingestion/generated/**
|
||||
# docs
|
||||
docs/generated/
|
||||
docs-website/versioned_docs/
|
||||
docs-website/versioned_sidebars/
|
||||
tmp*
|
||||
temp/**
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import json
|
||||
import os
|
||||
import tarfile
|
||||
import urllib.request
|
||||
import json
|
||||
|
||||
repo_url = "https://api.github.com/repos/datahub-project/static-assets"
|
||||
|
||||
@ -16,30 +16,36 @@ def download_file(url, destination):
|
||||
f.write(chunk)
|
||||
|
||||
|
||||
def fetch_tar_urls(repo_url, folder_path):
|
||||
def fetch_urls(repo_url: str, folder_path: str, file_format: str):
|
||||
api_url = f"{repo_url}/contents/{folder_path}"
|
||||
response = urllib.request.urlopen(api_url)
|
||||
data = response.read().decode('utf-8')
|
||||
tar_urls = [
|
||||
file["download_url"] for file in json.loads(data) if file["name"].endswith(".tar.gz")
|
||||
data = response.read().decode("utf-8")
|
||||
urls = [
|
||||
file["download_url"]
|
||||
for file in json.loads(data)
|
||||
if file["name"].endswith(file_format)
|
||||
]
|
||||
print(tar_urls)
|
||||
return tar_urls
|
||||
print(urls)
|
||||
return urls
|
||||
|
||||
|
||||
def main():
|
||||
folder_path = "versioned_docs"
|
||||
destination_dir = "versioned_docs"
|
||||
def extract_tar_file(destination_path):
|
||||
with tarfile.open(destination_path, "r:gz") as tar:
|
||||
tar.extractall()
|
||||
os.remove(destination_path)
|
||||
|
||||
|
||||
def download_versioned_docs(folder_path: str, destination_dir: str, file_format: str):
|
||||
if not os.path.exists(destination_dir):
|
||||
os.makedirs(destination_dir)
|
||||
|
||||
tar_urls = fetch_tar_urls(repo_url, folder_path)
|
||||
urls = fetch_urls(repo_url, folder_path, file_format)
|
||||
|
||||
for url in tar_urls:
|
||||
for url in urls:
|
||||
filename = os.path.basename(url)
|
||||
destination_path = os.path.join(destination_dir, filename)
|
||||
|
||||
version = '.'.join(filename.split('.')[:3])
|
||||
version = ".".join(filename.split(".")[:3])
|
||||
extracted_path = os.path.join(destination_dir, version)
|
||||
print("extracted_path", extracted_path)
|
||||
if os.path.exists(extracted_path):
|
||||
@ -48,13 +54,25 @@ def main():
|
||||
try:
|
||||
download_file(url, destination_path)
|
||||
print(f"Downloaded {filename} to {destination_dir}")
|
||||
with tarfile.open(destination_path, "r:gz") as tar:
|
||||
tar.extractall()
|
||||
os.remove(destination_path)
|
||||
if file_format == ".tar.gz":
|
||||
extract_tar_file(destination_path)
|
||||
except urllib.error.URLError as e:
|
||||
print(f"Error while downloading {filename}: {e}")
|
||||
continue
|
||||
|
||||
|
||||
def main():
|
||||
download_versioned_docs(
|
||||
folder_path="versioned_docs",
|
||||
destination_dir="versioned_docs",
|
||||
file_format=".tar.gz",
|
||||
)
|
||||
download_versioned_docs(
|
||||
folder_path="versioned_sidebars",
|
||||
destination_dir="versioned_sidebars",
|
||||
file_format=".json",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -1,594 +0,0 @@
|
||||
{
|
||||
"overviewSidebar": [
|
||||
{
|
||||
"label": "Getting Started",
|
||||
"type": "category",
|
||||
"collapsed": true,
|
||||
"items": [
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Introduction",
|
||||
"id": "docs/features"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Quickstart",
|
||||
"id": "docs/quickstart"
|
||||
},
|
||||
{
|
||||
"type": "link",
|
||||
"label": "Demo",
|
||||
"href": "https://demo.datahubproject.io/"
|
||||
},
|
||||
"docs/what-is-datahub/datahub-concepts",
|
||||
"docs/saas"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Integrations": [
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Introduction",
|
||||
"id": "metadata-ingestion/README"
|
||||
},
|
||||
{
|
||||
"Quickstart Guides": [
|
||||
{
|
||||
"BigQuery": [
|
||||
"docs/quick-ingestion-guides/bigquery/overview",
|
||||
"docs/quick-ingestion-guides/bigquery/setup",
|
||||
"docs/quick-ingestion-guides/bigquery/configuration"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Redshift": [
|
||||
"docs/quick-ingestion-guides/redshift/overview",
|
||||
"docs/quick-ingestion-guides/redshift/setup",
|
||||
"docs/quick-ingestion-guides/redshift/configuration"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Snowflake": [
|
||||
"docs/quick-ingestion-guides/snowflake/overview",
|
||||
"docs/quick-ingestion-guides/snowflake/setup",
|
||||
"docs/quick-ingestion-guides/snowflake/configuration"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Tableau": [
|
||||
"docs/quick-ingestion-guides/tableau/overview",
|
||||
"docs/quick-ingestion-guides/tableau/setup",
|
||||
"docs/quick-ingestion-guides/tableau/configuration"
|
||||
]
|
||||
},
|
||||
{
|
||||
"PowerBI": [
|
||||
"docs/quick-ingestion-guides/powerbi/overview",
|
||||
"docs/quick-ingestion-guides/powerbi/setup",
|
||||
"docs/quick-ingestion-guides/powerbi/configuration"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Sources": [
|
||||
{
|
||||
"type": "doc",
|
||||
"id": "docs/lineage/airflow",
|
||||
"label": "Airflow"
|
||||
},
|
||||
"metadata-integration/java/spark-lineage/README",
|
||||
"metadata-ingestion/integration_docs/great-expectations",
|
||||
"metadata-integration/java/datahub-protobuf/README",
|
||||
{
|
||||
"type": "autogenerated",
|
||||
"dirName": "docs/generated/ingestion/sources"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Sinks": [
|
||||
{
|
||||
"type": "autogenerated",
|
||||
"dirName": "metadata-ingestion/sink_docs"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Transformers": [
|
||||
"metadata-ingestion/docs/transformer/intro",
|
||||
"metadata-ingestion/docs/transformer/dataset_transformer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Advanced Guides": [
|
||||
{
|
||||
"Scheduling Ingestion": [
|
||||
"metadata-ingestion/schedule_docs/intro",
|
||||
"metadata-ingestion/schedule_docs/cron",
|
||||
"metadata-ingestion/schedule_docs/airflow",
|
||||
"metadata-ingestion/schedule_docs/kubernetes"
|
||||
]
|
||||
},
|
||||
"docs/platform-instances",
|
||||
"metadata-ingestion/docs/dev_guides/stateful",
|
||||
"metadata-ingestion/docs/dev_guides/classification",
|
||||
"metadata-ingestion/docs/dev_guides/add_stateful_ingestion_to_source",
|
||||
"metadata-ingestion/docs/dev_guides/sql_profiles"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Deployment": [
|
||||
"docs/deploy/aws",
|
||||
"docs/deploy/gcp",
|
||||
"docker/README",
|
||||
"docs/deploy/kubernetes",
|
||||
"docs/deploy/environment-vars",
|
||||
{
|
||||
"Authentication": [
|
||||
"docs/authentication/README",
|
||||
"docs/authentication/concepts",
|
||||
"docs/authentication/changing-default-credentials",
|
||||
"docs/authentication/guides/add-users",
|
||||
{
|
||||
"Frontend Authentication": [
|
||||
"docs/authentication/guides/jaas",
|
||||
{
|
||||
"OIDC Authentication": [
|
||||
"docs/authentication/guides/sso/configure-oidc-react",
|
||||
"docs/authentication/guides/sso/configure-oidc-react-google",
|
||||
"docs/authentication/guides/sso/configure-oidc-react-okta",
|
||||
"docs/authentication/guides/sso/configure-oidc-react-azure"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"docs/authentication/introducing-metadata-service-authentication",
|
||||
"docs/authentication/personal-access-tokens"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Authorization": [
|
||||
"docs/authorization/README",
|
||||
"docs/authorization/roles",
|
||||
"docs/authorization/policies",
|
||||
"docs/authorization/groups"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Advanced Guides": [
|
||||
"docs/how/delete-metadata",
|
||||
"docs/how/configuring-authorization-with-apache-ranger",
|
||||
"docs/how/backup-datahub",
|
||||
"docs/how/restore-indices",
|
||||
"docs/advanced/db-retention",
|
||||
"docs/advanced/monitoring",
|
||||
"docs/how/extract-container-logs",
|
||||
"docs/deploy/telemetry",
|
||||
"docs/how/kafka-config",
|
||||
"docs/deploy/confluent-cloud",
|
||||
"docs/advanced/no-code-upgrade",
|
||||
"docs/how/jattach-guide"
|
||||
]
|
||||
},
|
||||
"docs/how/updating-datahub"
|
||||
]
|
||||
},
|
||||
{
|
||||
"API": [
|
||||
"docs/api/datahub-apis",
|
||||
{
|
||||
"GraphQL API": [
|
||||
{
|
||||
"label": "Overview",
|
||||
"type": "doc",
|
||||
"id": "docs/api/graphql/overview"
|
||||
},
|
||||
{
|
||||
"Reference": [
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Queries",
|
||||
"id": "graphql/queries"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Mutations",
|
||||
"id": "graphql/mutations"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Objects",
|
||||
"id": "graphql/objects"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Inputs",
|
||||
"id": "graphql/inputObjects"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Interfaces",
|
||||
"id": "graphql/interfaces"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Unions",
|
||||
"id": "graphql/unions"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Enums",
|
||||
"id": "graphql/enums"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Scalars",
|
||||
"id": "graphql/scalars"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Guides": [
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "How To Set Up GraphQL",
|
||||
"id": "docs/api/graphql/how-to-set-up-graphql"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Getting Started With GraphQL",
|
||||
"id": "docs/api/graphql/getting-started"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Access Token Management",
|
||||
"id": "docs/api/graphql/token-management"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "OpenAPI",
|
||||
"id": "docs/api/openapi/openapi-usage-guide"
|
||||
},
|
||||
"docs/dev-guides/timeline",
|
||||
{
|
||||
"Rest.li API": [
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Rest.li API Guide",
|
||||
"id": "docs/api/restli/restli-overview"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Restore Indices",
|
||||
"id": "docs/api/restli/restore-indices"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Get Index Sizes",
|
||||
"id": "docs/api/restli/get-index-sizes"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Truncate Timeseries Aspect",
|
||||
"id": "docs/api/restli/truncate-time-series-aspect"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Get ElasticSearch Task Status Endpoint",
|
||||
"id": "docs/api/restli/get-elastic-task-status"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Evaluate Tests",
|
||||
"id": "docs/api/restli/evaluate-tests"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"label": "Aspect Versioning and Rest.li Modeling",
|
||||
"id": "docs/advanced/aspect-versioning"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Python SDK": [
|
||||
"metadata-ingestion/as-a-library",
|
||||
{
|
||||
"Python SDK Reference": [
|
||||
{
|
||||
"type": "autogenerated",
|
||||
"dirName": "python-sdk"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"metadata-integration/java/as-a-library",
|
||||
{
|
||||
"API and SDK Guides": [
|
||||
"docs/advanced/patch",
|
||||
"docs/api/tutorials/datasets",
|
||||
"docs/api/tutorials/lineage",
|
||||
"docs/api/tutorials/tags",
|
||||
"docs/api/tutorials/terms",
|
||||
"docs/api/tutorials/owners",
|
||||
"docs/api/tutorials/domains",
|
||||
"docs/api/tutorials/deprecation",
|
||||
"docs/api/tutorials/descriptions",
|
||||
"docs/api/tutorials/custom-properties",
|
||||
"docs/api/tutorials/ml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "category",
|
||||
"label": "DataHub CLI",
|
||||
"link": {
|
||||
"type": "doc",
|
||||
"id": "docs/cli"
|
||||
},
|
||||
"items": [
|
||||
"docs/datahub_lite"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "category",
|
||||
"label": "Datahub Actions",
|
||||
"link": {
|
||||
"type": "doc",
|
||||
"id": "docs/act-on-metadata"
|
||||
},
|
||||
"items": [
|
||||
"docs/actions/README",
|
||||
"docs/actions/quickstart",
|
||||
"docs/actions/concepts",
|
||||
{
|
||||
"Sources": [
|
||||
{
|
||||
"type": "autogenerated",
|
||||
"dirName": "docs/actions/sources"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Events": [
|
||||
{
|
||||
"type": "autogenerated",
|
||||
"dirName": "docs/actions/events"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Actions": [
|
||||
{
|
||||
"type": "autogenerated",
|
||||
"dirName": "docs/actions/actions"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Guides": [
|
||||
{
|
||||
"type": "autogenerated",
|
||||
"dirName": "docs/actions/guides"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Features": [
|
||||
"docs/ui-ingestion",
|
||||
"docs/how/search",
|
||||
"docs/schema-history",
|
||||
"docs/domains",
|
||||
"docs/dataproducts",
|
||||
"docs/glossary/business-glossary",
|
||||
"docs/tags",
|
||||
"docs/ownership/ownership-types",
|
||||
"docs/browse",
|
||||
"docs/authorization/access-policies-guide",
|
||||
"docs/features/dataset-usage-and-query-history",
|
||||
"docs/posts",
|
||||
"docs/sync-status",
|
||||
"docs/lineage/lineage-feature-guide",
|
||||
{
|
||||
"type": "doc",
|
||||
"id": "docs/tests/metadata-tests",
|
||||
"className": "saasOnly"
|
||||
},
|
||||
"docs/act-on-metadata/impact-analysis",
|
||||
{
|
||||
"Observability": [
|
||||
"docs/managed-datahub/observe/freshness-assertions"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Develop": [
|
||||
{
|
||||
"DataHub Metadata Model": [
|
||||
"docs/modeling/metadata-model",
|
||||
"docs/modeling/extending-the-metadata-model",
|
||||
"docs/what/mxe",
|
||||
{
|
||||
"Entities": [
|
||||
{
|
||||
"type": "autogenerated",
|
||||
"dirName": "docs/generated/metamodel/entities"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Architecture": [
|
||||
"docs/architecture/architecture",
|
||||
"docs/components",
|
||||
"docs/architecture/metadata-ingestion",
|
||||
"docs/architecture/metadata-serving",
|
||||
"docs/architecture/docker-containers"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Developing on DataHub": [
|
||||
"docs/developers",
|
||||
"docs/docker/development",
|
||||
"metadata-ingestion/developing",
|
||||
"docs/api/graphql/graphql-endpoint-development",
|
||||
{
|
||||
"Modules": [
|
||||
"datahub-web-react/README",
|
||||
"datahub-frontend/README",
|
||||
"datahub-graphql-core/README",
|
||||
"metadata-service/README",
|
||||
"metadata-jobs/mae-consumer-job/README",
|
||||
"metadata-jobs/mce-consumer-job/README"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"docs/plugins",
|
||||
{
|
||||
"Troubleshooting": [
|
||||
"docs/troubleshooting/quickstart",
|
||||
"docs/troubleshooting/build",
|
||||
"docs/troubleshooting/general"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Advanced": [
|
||||
"metadata-ingestion/docs/dev_guides/reporting_telemetry",
|
||||
"docs/advanced/mcp-mcl",
|
||||
"docker/datahub-upgrade/README",
|
||||
"docs/advanced/no-code-modeling",
|
||||
"datahub-web-react/src/app/analytics/README",
|
||||
"docs/how/migrating-graph-service-implementation",
|
||||
"docs/advanced/field-path-spec-v2",
|
||||
"metadata-ingestion/adding-source",
|
||||
"docs/how/add-custom-ingestion-source",
|
||||
"docs/how/add-custom-data-platform",
|
||||
"docs/advanced/browse-paths-upgrade",
|
||||
"docs/browseV2/browse-paths-v2"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Community": [
|
||||
"docs/slack",
|
||||
"docs/townhalls",
|
||||
"docs/townhall-history",
|
||||
"docs/CODE_OF_CONDUCT",
|
||||
"docs/CONTRIBUTING",
|
||||
"docs/links",
|
||||
"docs/rfc"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Managed DataHub": [
|
||||
"docs/managed-datahub/managed-datahub-overview",
|
||||
"docs/managed-datahub/welcome-acryl",
|
||||
{
|
||||
"type": "doc",
|
||||
"id": "docs/managed-datahub/saas-slack-setup",
|
||||
"className": "saasOnly"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"id": "docs/managed-datahub/approval-workflows",
|
||||
"className": "saasOnly"
|
||||
},
|
||||
{
|
||||
"Metadata Ingestion With Acryl": [
|
||||
"docs/managed-datahub/metadata-ingestion-with-acryl/ingestion"
|
||||
]
|
||||
},
|
||||
{
|
||||
"DataHub API": [
|
||||
{
|
||||
"type": "doc",
|
||||
"id": "docs/managed-datahub/datahub-api/entity-events-api",
|
||||
"className": "saasOnly"
|
||||
},
|
||||
{
|
||||
"GraphQL API": [
|
||||
"docs/managed-datahub/datahub-api/graphql-api/getting-started",
|
||||
{
|
||||
"type": "doc",
|
||||
"id": "docs/managed-datahub/datahub-api/graphql-api/incidents-api-beta",
|
||||
"className": "saasOnly"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Integrations": [
|
||||
{
|
||||
"type": "doc",
|
||||
"id": "docs/managed-datahub/integrations/aws-privatelink",
|
||||
"className": "saasOnly"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"id": "docs/managed-datahub/integrations/oidc-sso-integration",
|
||||
"className": "saasOnly"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Operator Guide": [
|
||||
{
|
||||
"type": "doc",
|
||||
"id": "docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor-on-aws",
|
||||
"className": "saasOnly"
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"id": "docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge",
|
||||
"className": "saasOnly"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "doc",
|
||||
"id": "docs/managed-datahub/chrome-extension",
|
||||
"className": "saasOnly"
|
||||
},
|
||||
{
|
||||
"Managed DataHub Release History": [
|
||||
"docs/managed-datahub/release-notes/v_0_2_10",
|
||||
"docs/managed-datahub/release-notes/v_0_2_9",
|
||||
"docs/managed-datahub/release-notes/v_0_2_8",
|
||||
"docs/managed-datahub/release-notes/v_0_2_7",
|
||||
"docs/managed-datahub/release-notes/v_0_2_6",
|
||||
"docs/managed-datahub/release-notes/v_0_2_5",
|
||||
"docs/managed-datahub/release-notes/v_0_2_4",
|
||||
"docs/managed-datahub/release-notes/v_0_2_3",
|
||||
"docs/managed-datahub/release-notes/v_0_2_2",
|
||||
"docs/managed-datahub/release-notes/v_0_2_1",
|
||||
"docs/managed-datahub/release-notes/v_0_2_0",
|
||||
"docs/managed-datahub/release-notes/v_0_1_73",
|
||||
"docs/managed-datahub/release-notes/v_0_1_72",
|
||||
"docs/managed-datahub/release-notes/v_0_1_70",
|
||||
"docs/managed-datahub/release-notes/v_0_1_69"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Release History": [
|
||||
"releases"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user