feat(docs-website): add vercel preview environment (#7644)

.github/workflows/documentation.yml

@@ -32,6 +32,7 @@ jobs:
       - name: Build Docs
         run: |
           ./gradlew --info docs-website:build
+
       - name: Deploy
         if: github.event_name == 'push' && github.repository == 'datahub-project/datahub'
         uses: peaceiris/actions-gh-pages@v3

.gitignore

@@ -78,3 +78,4 @@ datahub-frontend/public/**
 metadata-service/plugin/src/test/resources/sample-plugins/**
 
 smoke-test/rollback-reports
+.vercel

docs-website/.gitignore

@@ -31,3 +31,4 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 .vscode
+.vercel

docs-website/generateDocsDir.ts

@@ -592,6 +592,7 @@ function copy_python_wheels(): void {
     "docs/actions/sources",
     "docs/actions/guides",
     "metadata-ingestion/archived",
+    "metadata-ingestion/sink_docs",
     "docs/what",
     "docs/wip",
   ];

docs-website/sidebars.js

@@ -1,34 +1,3 @@
-const fs = require("fs");
-
-function list_ids_in_directory(directory, hardcoded_labels) {
-  if (hardcoded_labels === undefined) {
-    hardcoded_labels = {};
-  }
-
-  const files = fs.readdirSync(`../${directory}`).sort();
-  let ids = [];
-  for (const name of files) {
-    if (fs.lstatSync(`../${directory}/${name}`).isDirectory()) {
-      // Recurse into the directory.
-      const inner_ids = list_ids_in_directory(`${directory}/${name}`);
-      ids = ids.concat(inner_ids);
-    } else {
-      if (name.endsWith(".md")) {
-        const slug = name.replace(/\.md$/, "");
-        const id = `${directory}/${slug}`;
-
-        if (id in hardcoded_labels) {
-          label = hardcoded_labels[id];
-          ids.push({ type: "doc", id, label });
-        } else {
-          ids.push({ type: "doc", id });
-        }
-      }
-    }
-  }
-  return ids;
-}
-
 // note: to handle errors where you don't want a markdown file in the sidebar, add it as a comment.
 // this will fix errors like `Error: File not accounted for in sidebar: ...`
 module.exports = {
@@ -142,7 +111,12 @@ module.exports = {
           ],
         },
         {
-          Sinks: list_ids_in_directory("metadata-ingestion/sink_docs"),
+          Sinks: [
+            {
+              type: "autogenerated",
+              dirName: "metadata-ingestion/sink_docs",
+            },
+          ],
         },
         {
           Transformers: [

metadata-ingestion/scripts/install_deps.sh

@@ -4,10 +4,23 @@ set -euxo pipefail
 if [ "$(uname)" == "Darwin" ]; then
     brew install librdkafka
 else
-    sudo apt-get update && sudo apt-get install -y \
+    sudo_cmd=""
+    if command -v sudo; then
+        sudo_cmd="sudo"
+    fi
+
+    if command -v yum; then
+        $sudo_cmd yum install -y \
+            librdkafka-devel \
+            openldap-devel \
+            cyrus-sasl-devel \
+            openldap-clients
+    else
+        $sudo_cmd apt-get update && $sudo_cmd apt-get install -y \
             librdkafka-dev \
             python3-ldap \
             libldap2-dev \
             libsasl2-dev \
             ldap-utils
+    fi
 fi

vercel.json

@@ -0,0 +1,9 @@
+{
+  "buildCommand": "./gradlew :docs-website:build",
+  "github": {
+    "silent": true,
+    "autoJobCancelation": true
+  },
+  "installCommand": "amazon-linux-extras install python3.8 && py3=\"$(which python3)\" && rm \"$py3\" && ln \"$(which python3.8)\" \"$py3\" && ./metadata-ingestion/scripts/install_deps.sh && yum install -y gcc python38-devel",
+  "outputDirectory": "docs-website/build"
+}