feat(docs): move versioned_sidebars to static-assets (#8743)

This commit is contained in:
Hyejin Yoon 2023-08-30 09:27:38 +09:00 committed by GitHub
parent 97a77d8d0b
commit bebee88100
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 35 additions and 610 deletions

1
.gitignore vendored
View File

@ -70,6 +70,7 @@ metadata-ingestion/generated/**
# docs
docs/generated/
docs-website/versioned_docs/
docs-website/versioned_sidebars/
tmp*
temp/**

View File

@ -1,7 +1,7 @@
import json
import os
import tarfile
import urllib.request
import json
repo_url = "https://api.github.com/repos/datahub-project/static-assets"
@ -16,30 +16,36 @@ def download_file(url, destination):
f.write(chunk)
def fetch_tar_urls(repo_url, folder_path):
def fetch_urls(repo_url: str, folder_path: str, file_format: str):
api_url = f"{repo_url}/contents/{folder_path}"
response = urllib.request.urlopen(api_url)
data = response.read().decode('utf-8')
tar_urls = [
file["download_url"] for file in json.loads(data) if file["name"].endswith(".tar.gz")
data = response.read().decode("utf-8")
urls = [
file["download_url"]
for file in json.loads(data)
if file["name"].endswith(file_format)
]
print(tar_urls)
return tar_urls
print(urls)
return urls
def main():
folder_path = "versioned_docs"
destination_dir = "versioned_docs"
def extract_tar_file(destination_path):
with tarfile.open(destination_path, "r:gz") as tar:
tar.extractall()
os.remove(destination_path)
def download_versioned_docs(folder_path: str, destination_dir: str, file_format: str):
if not os.path.exists(destination_dir):
os.makedirs(destination_dir)
tar_urls = fetch_tar_urls(repo_url, folder_path)
urls = fetch_urls(repo_url, folder_path, file_format)
for url in tar_urls:
for url in urls:
filename = os.path.basename(url)
destination_path = os.path.join(destination_dir, filename)
version = '.'.join(filename.split('.')[:3])
version = ".".join(filename.split(".")[:3])
extracted_path = os.path.join(destination_dir, version)
print("extracted_path", extracted_path)
if os.path.exists(extracted_path):
@ -48,13 +54,25 @@ def main():
try:
download_file(url, destination_path)
print(f"Downloaded {filename} to {destination_dir}")
with tarfile.open(destination_path, "r:gz") as tar:
tar.extractall()
os.remove(destination_path)
if file_format == ".tar.gz":
extract_tar_file(destination_path)
except urllib.error.URLError as e:
print(f"Error while downloading {filename}: {e}")
continue
def main():
download_versioned_docs(
folder_path="versioned_docs",
destination_dir="versioned_docs",
file_format=".tar.gz",
)
download_versioned_docs(
folder_path="versioned_sidebars",
destination_dir="versioned_sidebars",
file_format=".json",
)
if __name__ == "__main__":
main()

View File

@ -1,594 +0,0 @@
{
"overviewSidebar": [
{
"label": "Getting Started",
"type": "category",
"collapsed": true,
"items": [
{
"type": "doc",
"label": "Introduction",
"id": "docs/features"
},
{
"type": "doc",
"label": "Quickstart",
"id": "docs/quickstart"
},
{
"type": "link",
"label": "Demo",
"href": "https://demo.datahubproject.io/"
},
"docs/what-is-datahub/datahub-concepts",
"docs/saas"
]
},
{
"Integrations": [
{
"type": "doc",
"label": "Introduction",
"id": "metadata-ingestion/README"
},
{
"Quickstart Guides": [
{
"BigQuery": [
"docs/quick-ingestion-guides/bigquery/overview",
"docs/quick-ingestion-guides/bigquery/setup",
"docs/quick-ingestion-guides/bigquery/configuration"
]
},
{
"Redshift": [
"docs/quick-ingestion-guides/redshift/overview",
"docs/quick-ingestion-guides/redshift/setup",
"docs/quick-ingestion-guides/redshift/configuration"
]
},
{
"Snowflake": [
"docs/quick-ingestion-guides/snowflake/overview",
"docs/quick-ingestion-guides/snowflake/setup",
"docs/quick-ingestion-guides/snowflake/configuration"
]
},
{
"Tableau": [
"docs/quick-ingestion-guides/tableau/overview",
"docs/quick-ingestion-guides/tableau/setup",
"docs/quick-ingestion-guides/tableau/configuration"
]
},
{
"PowerBI": [
"docs/quick-ingestion-guides/powerbi/overview",
"docs/quick-ingestion-guides/powerbi/setup",
"docs/quick-ingestion-guides/powerbi/configuration"
]
}
]
},
{
"Sources": [
{
"type": "doc",
"id": "docs/lineage/airflow",
"label": "Airflow"
},
"metadata-integration/java/spark-lineage/README",
"metadata-ingestion/integration_docs/great-expectations",
"metadata-integration/java/datahub-protobuf/README",
{
"type": "autogenerated",
"dirName": "docs/generated/ingestion/sources"
}
]
},
{
"Sinks": [
{
"type": "autogenerated",
"dirName": "metadata-ingestion/sink_docs"
}
]
},
{
"Transformers": [
"metadata-ingestion/docs/transformer/intro",
"metadata-ingestion/docs/transformer/dataset_transformer"
]
},
{
"Advanced Guides": [
{
"Scheduling Ingestion": [
"metadata-ingestion/schedule_docs/intro",
"metadata-ingestion/schedule_docs/cron",
"metadata-ingestion/schedule_docs/airflow",
"metadata-ingestion/schedule_docs/kubernetes"
]
},
"docs/platform-instances",
"metadata-ingestion/docs/dev_guides/stateful",
"metadata-ingestion/docs/dev_guides/classification",
"metadata-ingestion/docs/dev_guides/add_stateful_ingestion_to_source",
"metadata-ingestion/docs/dev_guides/sql_profiles"
]
}
]
},
{
"Deployment": [
"docs/deploy/aws",
"docs/deploy/gcp",
"docker/README",
"docs/deploy/kubernetes",
"docs/deploy/environment-vars",
{
"Authentication": [
"docs/authentication/README",
"docs/authentication/concepts",
"docs/authentication/changing-default-credentials",
"docs/authentication/guides/add-users",
{
"Frontend Authentication": [
"docs/authentication/guides/jaas",
{
"OIDC Authentication": [
"docs/authentication/guides/sso/configure-oidc-react",
"docs/authentication/guides/sso/configure-oidc-react-google",
"docs/authentication/guides/sso/configure-oidc-react-okta",
"docs/authentication/guides/sso/configure-oidc-react-azure"
]
}
]
},
"docs/authentication/introducing-metadata-service-authentication",
"docs/authentication/personal-access-tokens"
]
},
{
"Authorization": [
"docs/authorization/README",
"docs/authorization/roles",
"docs/authorization/policies",
"docs/authorization/groups"
]
},
{
"Advanced Guides": [
"docs/how/delete-metadata",
"docs/how/configuring-authorization-with-apache-ranger",
"docs/how/backup-datahub",
"docs/how/restore-indices",
"docs/advanced/db-retention",
"docs/advanced/monitoring",
"docs/how/extract-container-logs",
"docs/deploy/telemetry",
"docs/how/kafka-config",
"docs/deploy/confluent-cloud",
"docs/advanced/no-code-upgrade",
"docs/how/jattach-guide"
]
},
"docs/how/updating-datahub"
]
},
{
"API": [
"docs/api/datahub-apis",
{
"GraphQL API": [
{
"label": "Overview",
"type": "doc",
"id": "docs/api/graphql/overview"
},
{
"Reference": [
{
"type": "doc",
"label": "Queries",
"id": "graphql/queries"
},
{
"type": "doc",
"label": "Mutations",
"id": "graphql/mutations"
},
{
"type": "doc",
"label": "Objects",
"id": "graphql/objects"
},
{
"type": "doc",
"label": "Inputs",
"id": "graphql/inputObjects"
},
{
"type": "doc",
"label": "Interfaces",
"id": "graphql/interfaces"
},
{
"type": "doc",
"label": "Unions",
"id": "graphql/unions"
},
{
"type": "doc",
"label": "Enums",
"id": "graphql/enums"
},
{
"type": "doc",
"label": "Scalars",
"id": "graphql/scalars"
}
]
},
{
"Guides": [
{
"type": "doc",
"label": "How To Set Up GraphQL",
"id": "docs/api/graphql/how-to-set-up-graphql"
},
{
"type": "doc",
"label": "Getting Started With GraphQL",
"id": "docs/api/graphql/getting-started"
},
{
"type": "doc",
"label": "Access Token Management",
"id": "docs/api/graphql/token-management"
}
]
}
]
},
{
"type": "doc",
"label": "OpenAPI",
"id": "docs/api/openapi/openapi-usage-guide"
},
"docs/dev-guides/timeline",
{
"Rest.li API": [
{
"type": "doc",
"label": "Rest.li API Guide",
"id": "docs/api/restli/restli-overview"
},
{
"type": "doc",
"label": "Restore Indices",
"id": "docs/api/restli/restore-indices"
},
{
"type": "doc",
"label": "Get Index Sizes",
"id": "docs/api/restli/get-index-sizes"
},
{
"type": "doc",
"label": "Truncate Timeseries Aspect",
"id": "docs/api/restli/truncate-time-series-aspect"
},
{
"type": "doc",
"label": "Get ElasticSearch Task Status Endpoint",
"id": "docs/api/restli/get-elastic-task-status"
},
{
"type": "doc",
"label": "Evaluate Tests",
"id": "docs/api/restli/evaluate-tests"
},
{
"type": "doc",
"label": "Aspect Versioning and Rest.li Modeling",
"id": "docs/advanced/aspect-versioning"
}
]
},
{
"Python SDK": [
"metadata-ingestion/as-a-library",
{
"Python SDK Reference": [
{
"type": "autogenerated",
"dirName": "python-sdk"
}
]
}
]
},
"metadata-integration/java/as-a-library",
{
"API and SDK Guides": [
"docs/advanced/patch",
"docs/api/tutorials/datasets",
"docs/api/tutorials/lineage",
"docs/api/tutorials/tags",
"docs/api/tutorials/terms",
"docs/api/tutorials/owners",
"docs/api/tutorials/domains",
"docs/api/tutorials/deprecation",
"docs/api/tutorials/descriptions",
"docs/api/tutorials/custom-properties",
"docs/api/tutorials/ml"
]
},
{
"type": "category",
"label": "DataHub CLI",
"link": {
"type": "doc",
"id": "docs/cli"
},
"items": [
"docs/datahub_lite"
]
},
{
"type": "category",
"label": "Datahub Actions",
"link": {
"type": "doc",
"id": "docs/act-on-metadata"
},
"items": [
"docs/actions/README",
"docs/actions/quickstart",
"docs/actions/concepts",
{
"Sources": [
{
"type": "autogenerated",
"dirName": "docs/actions/sources"
}
]
},
{
"Events": [
{
"type": "autogenerated",
"dirName": "docs/actions/events"
}
]
},
{
"Actions": [
{
"type": "autogenerated",
"dirName": "docs/actions/actions"
}
]
},
{
"Guides": [
{
"type": "autogenerated",
"dirName": "docs/actions/guides"
}
]
}
]
}
]
},
{
"Features": [
"docs/ui-ingestion",
"docs/how/search",
"docs/schema-history",
"docs/domains",
"docs/dataproducts",
"docs/glossary/business-glossary",
"docs/tags",
"docs/ownership/ownership-types",
"docs/browse",
"docs/authorization/access-policies-guide",
"docs/features/dataset-usage-and-query-history",
"docs/posts",
"docs/sync-status",
"docs/lineage/lineage-feature-guide",
{
"type": "doc",
"id": "docs/tests/metadata-tests",
"className": "saasOnly"
},
"docs/act-on-metadata/impact-analysis",
{
"Observability": [
"docs/managed-datahub/observe/freshness-assertions"
]
}
]
},
{
"Develop": [
{
"DataHub Metadata Model": [
"docs/modeling/metadata-model",
"docs/modeling/extending-the-metadata-model",
"docs/what/mxe",
{
"Entities": [
{
"type": "autogenerated",
"dirName": "docs/generated/metamodel/entities"
}
]
}
]
},
{
"Architecture": [
"docs/architecture/architecture",
"docs/components",
"docs/architecture/metadata-ingestion",
"docs/architecture/metadata-serving",
"docs/architecture/docker-containers"
]
},
{
"Developing on DataHub": [
"docs/developers",
"docs/docker/development",
"metadata-ingestion/developing",
"docs/api/graphql/graphql-endpoint-development",
{
"Modules": [
"datahub-web-react/README",
"datahub-frontend/README",
"datahub-graphql-core/README",
"metadata-service/README",
"metadata-jobs/mae-consumer-job/README",
"metadata-jobs/mce-consumer-job/README"
]
}
]
},
"docs/plugins",
{
"Troubleshooting": [
"docs/troubleshooting/quickstart",
"docs/troubleshooting/build",
"docs/troubleshooting/general"
]
},
{
"Advanced": [
"metadata-ingestion/docs/dev_guides/reporting_telemetry",
"docs/advanced/mcp-mcl",
"docker/datahub-upgrade/README",
"docs/advanced/no-code-modeling",
"datahub-web-react/src/app/analytics/README",
"docs/how/migrating-graph-service-implementation",
"docs/advanced/field-path-spec-v2",
"metadata-ingestion/adding-source",
"docs/how/add-custom-ingestion-source",
"docs/how/add-custom-data-platform",
"docs/advanced/browse-paths-upgrade",
"docs/browseV2/browse-paths-v2"
]
}
]
},
{
"Community": [
"docs/slack",
"docs/townhalls",
"docs/townhall-history",
"docs/CODE_OF_CONDUCT",
"docs/CONTRIBUTING",
"docs/links",
"docs/rfc"
]
},
{
"Managed DataHub": [
"docs/managed-datahub/managed-datahub-overview",
"docs/managed-datahub/welcome-acryl",
{
"type": "doc",
"id": "docs/managed-datahub/saas-slack-setup",
"className": "saasOnly"
},
{
"type": "doc",
"id": "docs/managed-datahub/approval-workflows",
"className": "saasOnly"
},
{
"Metadata Ingestion With Acryl": [
"docs/managed-datahub/metadata-ingestion-with-acryl/ingestion"
]
},
{
"DataHub API": [
{
"type": "doc",
"id": "docs/managed-datahub/datahub-api/entity-events-api",
"className": "saasOnly"
},
{
"GraphQL API": [
"docs/managed-datahub/datahub-api/graphql-api/getting-started",
{
"type": "doc",
"id": "docs/managed-datahub/datahub-api/graphql-api/incidents-api-beta",
"className": "saasOnly"
}
]
}
]
},
{
"Integrations": [
{
"type": "doc",
"id": "docs/managed-datahub/integrations/aws-privatelink",
"className": "saasOnly"
},
{
"type": "doc",
"id": "docs/managed-datahub/integrations/oidc-sso-integration",
"className": "saasOnly"
}
]
},
{
"Operator Guide": [
{
"type": "doc",
"id": "docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor-on-aws",
"className": "saasOnly"
},
{
"type": "doc",
"id": "docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge",
"className": "saasOnly"
}
]
},
{
"type": "doc",
"id": "docs/managed-datahub/chrome-extension",
"className": "saasOnly"
},
{
"Managed DataHub Release History": [
"docs/managed-datahub/release-notes/v_0_2_10",
"docs/managed-datahub/release-notes/v_0_2_9",
"docs/managed-datahub/release-notes/v_0_2_8",
"docs/managed-datahub/release-notes/v_0_2_7",
"docs/managed-datahub/release-notes/v_0_2_6",
"docs/managed-datahub/release-notes/v_0_2_5",
"docs/managed-datahub/release-notes/v_0_2_4",
"docs/managed-datahub/release-notes/v_0_2_3",
"docs/managed-datahub/release-notes/v_0_2_2",
"docs/managed-datahub/release-notes/v_0_2_1",
"docs/managed-datahub/release-notes/v_0_2_0",
"docs/managed-datahub/release-notes/v_0_1_73",
"docs/managed-datahub/release-notes/v_0_1_72",
"docs/managed-datahub/release-notes/v_0_1_70",
"docs/managed-datahub/release-notes/v_0_1_69"
]
}
]
},
{
"Release History": [
"releases"
]
}
]
}