build(docs-website): make codegen script idempotent (#5620)

docs-website/.gitignore

@@ -1,6 +1,7 @@
 !.gitignore
 
-/genDocs/*
+/docs
+/genDocs
 
 # Generated GraphQL
 /graphql/combined.graphql

docs-website/README.md

@@ -1,6 +1,6 @@
 # Website
 
-This website is built using [Docusaurus 2](https://v2.docusaurus.io/), a modern static website generator.
+This website is built using [Docusaurus](https://docusaurus.io/), a modern static website generator.
 
 ## Installation
 
@@ -11,32 +11,27 @@ yarn install
 ## Local Development
 
 ```console
+../gradlew yarnStart
+
+# You may also have success running the underlying commands manually.
 yarn run lint
 yarn run generate
 yarn run start
 ```
 
-This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server.
+This command starts a local development server and open up a browser window.
 
 ## Build
 
 ```console
-yarn build
+../gradlew yarnBuild
 ```
 
-This command generates static content into the `build` directory and can be served using any static contents hosting service.
-
-## Deployment
-
-```console
-GIT_USER=<Your GitHub username> USE_SSH=true yarn deploy
-```
-
-If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch.
+This command generates static content into the `dist` directory and can be served using any static contents hosting service. You can preview the built static site using `../gradlew serve`, although we're recommend using the local development instructions locally.
 
 ## Generating GraphQL API Docs
 
-To regenerate GraphQL API docs, simplym rebuild the docs-website directory. 
+To regenerate GraphQL API docs, simply rebuild the docs-website directory. 
 
 ```console
 ./gradlew docs-website:build

docs-website/build.gradle

@@ -68,17 +68,13 @@ task yarnInstall(type: YarnTask) {
   }
 }
 
-task generateGraphQLDocumentation(type: YarnTask, dependsOn: [yarnInstall, generateGraphQLSchema] ) {
-  args = ['docusaurus', 'docs:generate:graphql']
-}
-
-task yarnGenerate(type: YarnTask, dependsOn: [yarnInstall, generateGraphQLDocumentation, generateJsonSchema, ':metadata-ingestion:modelDocGen', ':metadata-ingestion:docGen'] ) {
+task yarnGenerate(type: YarnTask, dependsOn: [yarnInstall, generateGraphQLSchema, generateJsonSchema, ':metadata-ingestion:modelDocGen', ':metadata-ingestion:docGen'] ) {
   inputs.files(projectMdFiles)
   outputs.cacheIf { true }
   args = ['run', 'generate']
 }
 
-task yarnServe(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
+task yarnStart(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
   args = ['run', 'start']
 }
 

docs-website/docs/.gitignore

@@ -1,4 +0,0 @@
-# Ignore everything in this directory
-*
-# Except this file
-!.gitignore

docs-website/genDocs/.gitignore

@@ -1,4 +0,0 @@
-# Ignore everything in this directory
-*
-# Except this file
-!.gitignore

docs-website/generateDocsDir.ts

@@ -84,9 +84,26 @@ function list_markdown_files(): string[] {
       .filter((filepath) => !all_generated_markdown_files.includes(filepath));
 
     if (untracked_files.length > 0) {
-      console.log(`Including untracked files in docs list: ${untracked_files}`);
+      console.log(
+        `Including untracked files in docs list: [${untracked_files}]`
+      );
       all_markdown_files = [...all_markdown_files, ...untracked_files];
     }
+
+    // But we should also exclude any files that have been deleted.
+    const deleted_files = execSync(
+      "(git ls-files --full-name --deleted --exclude-standard .. | grep '.md$') || true"
+    )
+      .toString()
+      .trim()
+      .split("\n");
+
+    if (deleted_files.length > 0) {
+      console.log(`Removing deleted files from docs list: [${deleted_files}]`);
+      all_markdown_files = all_markdown_files.filter(
+        (filepath) => !deleted_files.includes(filepath)
+      );
+    }
   }
 
   const filter_patterns = [
@@ -453,7 +470,7 @@ custom_edit_url: https://github.com/datahub-project/datahub/blob/master/docs-web
   for (const release of releases_list.data) {
     let body: string;
     if (releaseNoteVersions.has(release.tag_name)) {
-      body = release.body;
+      body = release.body ?? "";
       body = markdown_sanitize_and_linkify(body);
 
       // Redo the heading levels. First we find the min heading level, and then

docs-website/package.json

@@ -10,7 +10,8 @@
     "deploy": "docusaurus deploy",
     "serve": "docusaurus serve",
     "clear": "docusaurus clear && rm -rf genDocs/*",
-    "generate": "rm -rf genDocs/* && ts-node -O '{ \"lib\": [\"es2020\"], \"target\": \"es6\" }' generateDocsDir.ts && mv -v docs/* genDocs/",
+    "generate-graphql-internal": "docusaurus docs:generate:graphql",
+    "generate": "rm -rf docs genDocs && mkdir docs genDocs && yarn generate-graphql-internal && ts-node -O '{ \"lib\": [\"es2020\"], \"target\": \"es6\" }' generateDocsDir.ts && mv -v docs/* genDocs/ && rmdir docs",
     "lint": "prettier -w generateDocsDir.ts sidebars.js src/pages/index.js",
     "lint-check": "prettier -l generateDocsDir.ts sidebars.js src/pages/index.js",
     "lint-fix": "prettier --write generateDocsDir.ts sidebars.js src/pages/index.js"