diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 4d1794b1a2..1cbc65f2b6 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -32,6 +32,7 @@ jobs: - name: Build Docs run: | ./gradlew --info docs-website:build + - name: Deploy if: github.event_name == 'push' && github.repository == 'datahub-project/datahub' uses: peaceiris/actions-gh-pages@v3 diff --git a/.gitignore b/.gitignore index 01638438f0..c4c6b8b941 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,4 @@ datahub-frontend/public/** metadata-service/plugin/src/test/resources/sample-plugins/** smoke-test/rollback-reports +.vercel diff --git a/docs-website/.gitignore b/docs-website/.gitignore index 2c52c6154b..9ffda961bf 100644 --- a/docs-website/.gitignore +++ b/docs-website/.gitignore @@ -31,3 +31,4 @@ npm-debug.log* yarn-debug.log* yarn-error.log* .vscode +.vercel diff --git a/docs-website/generateDocsDir.ts b/docs-website/generateDocsDir.ts index 8e8e007bd5..df5bb226ba 100644 --- a/docs-website/generateDocsDir.ts +++ b/docs-website/generateDocsDir.ts @@ -592,6 +592,7 @@ function copy_python_wheels(): void { "docs/actions/sources", "docs/actions/guides", "metadata-ingestion/archived", + "metadata-ingestion/sink_docs", "docs/what", "docs/wip", ]; diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 1948e793f9..5a2a217dbd 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -1,34 +1,3 @@ -const fs = require("fs"); - -function list_ids_in_directory(directory, hardcoded_labels) { - if (hardcoded_labels === undefined) { - hardcoded_labels = {}; - } - - const files = fs.readdirSync(`../${directory}`).sort(); - let ids = []; - for (const name of files) { - if (fs.lstatSync(`../${directory}/${name}`).isDirectory()) { - // Recurse into the directory. - const inner_ids = list_ids_in_directory(`${directory}/${name}`); - ids = ids.concat(inner_ids); - } else { - if (name.endsWith(".md")) { - const slug = name.replace(/\.md$/, ""); - const id = `${directory}/${slug}`; - - if (id in hardcoded_labels) { - label = hardcoded_labels[id]; - ids.push({ type: "doc", id, label }); - } else { - ids.push({ type: "doc", id }); - } - } - } - } - return ids; -} - // note: to handle errors where you don't want a markdown file in the sidebar, add it as a comment. // this will fix errors like `Error: File not accounted for in sidebar: ...` module.exports = { @@ -142,7 +111,12 @@ module.exports = { ], }, { - Sinks: list_ids_in_directory("metadata-ingestion/sink_docs"), + Sinks: [ + { + type: "autogenerated", + dirName: "metadata-ingestion/sink_docs", + }, + ], }, { Transformers: [ diff --git a/metadata-ingestion/scripts/install_deps.sh b/metadata-ingestion/scripts/install_deps.sh index f7d1799e6d..d85082f396 100755 --- a/metadata-ingestion/scripts/install_deps.sh +++ b/metadata-ingestion/scripts/install_deps.sh @@ -4,10 +4,23 @@ set -euxo pipefail if [ "$(uname)" == "Darwin" ]; then brew install librdkafka else - sudo apt-get update && sudo apt-get install -y \ - librdkafka-dev \ - python3-ldap \ - libldap2-dev \ - libsasl2-dev \ - ldap-utils + sudo_cmd="" + if command -v sudo; then + sudo_cmd="sudo" + fi + + if command -v yum; then + $sudo_cmd yum install -y \ + librdkafka-devel \ + openldap-devel \ + cyrus-sasl-devel \ + openldap-clients + else + $sudo_cmd apt-get update && $sudo_cmd apt-get install -y \ + librdkafka-dev \ + python3-ldap \ + libldap2-dev \ + libsasl2-dev \ + ldap-utils + fi fi diff --git a/vercel.json b/vercel.json new file mode 100644 index 0000000000..dddc6adc8f --- /dev/null +++ b/vercel.json @@ -0,0 +1,9 @@ +{ + "buildCommand": "./gradlew :docs-website:build", + "github": { + "silent": true, + "autoJobCancelation": true + }, + "installCommand": "amazon-linux-extras install python3.8 && py3=\"$(which python3)\" && rm \"$py3\" && ln \"$(which python3.8)\" \"$py3\" && ./metadata-ingestion/scripts/install_deps.sh && yum install -y gcc python38-devel", + "outputDirectory": "docs-website/build" +}