feat(docs): automatically populate sidebar with RFCs (#2191)

docs-website/generateDocsDir.ts

@@ -12,11 +12,30 @@ const GITHUB_BROWSE_URL = "https://github.com/linkedin/datahub/blob/master";
 
 const SIDEBARS_DEF_PATH = "./sidebars.js";
 const sidebars = require(SIDEBARS_DEF_PATH);
+const sidebars_json = JSON.stringify(sidebars);
+const sidebars_text = fs.readFileSync(SIDEBARS_DEF_PATH).toString();
+console.log(sidebars_json);
 
 function actually_in_sidebar(filepath: string): boolean {
-  const slug = get_slug(filepath);
-  const json = JSON.stringify(sidebars);
-  return json.indexOf(slug) < 0;
+  const doc_id = get_id(filepath);
+  return sidebars_json.indexOf(`"${doc_id}"`) >= 0;
+}
+
+function accounted_for_in_sidebar(filepath: string): boolean {
+  if (actually_in_sidebar(filepath)) {
+    return true;
+  }
+
+  // If we want to explicitly not include a doc in the sidebar but still have it
+  // available via a direct link, then keeping it commented out in the sidebar
+  // serves this purpose. To make it work, we search the text of the sidebar JS
+  // file.
+  const doc_id = get_id(filepath);
+  if (sidebars_text.indexOf(`"${doc_id}"`) >= 0) {
+    return true;
+  }
+
+  return false;
 }
 
 function list_markdown_files(): string[] {
@@ -81,6 +100,15 @@ const hardcoded_titles = {
   "docs/demo.md": "Demo",
 };
 
+// FIXME: Eventually, we'd like to fix all of the broken links within these files.
+const allowed_broken_links = [
+  "docs/architecture/metadata-serving.md",
+  "docs/developers.md",
+  "docs/how/customize-elasticsearch-query-template.md",
+  "docs/how/graph-onboarding.md",
+  "docs/how/search-onboarding.md",
+];
+
 function markdown_guess_title(
   contents: matter.GrayMatterFile<string>,
   filepath: string
@@ -177,11 +205,16 @@ function new_url(original: string, filepath: string): string {
     // A reference to a file or directory in the Github repo.
     const relation = path.dirname(filepath);
     const updated_path = path.normalize(`${relation}/${original}`);
-    if (!fs.existsSync(`../${updated_path}`) && actually_in_sidebar(filepath)) {
+    const check_path = updated_path.replace(/#.+$/, "");
+    if (
+      !fs.existsSync(`../${check_path}`) &&
+      actually_in_sidebar(filepath) &&
+      !allowed_broken_links.includes(filepath)
+    ) {
       // Detects when the path is a dangling reference, according to the locally
       // checked out repo.
       throw new Error(
-        `broken github repo link: ${updated_path} in ${filepath}`
+        `broken github repo link to ${updated_path} in ${filepath}`
       );
     }
     const updated_url = `${GITHUB_BROWSE_URL}/${updated_path}`;
@@ -258,12 +291,12 @@ for (const filepath of markdown_files) {
   fs.writeFileSync(outpath, contents.stringify(""));
 }
 
-// Output a list of all docs which are not included in a sidebar.
+// Error if a doc is not accounted for in a sidebar.
 const sidebar = fs.readFileSync(SIDEBARS_DEF_PATH).toString();
 for (const filepath of markdown_files) {
-  const doc_id = get_id(filepath);
-
-  if (sidebar.indexOf(`"${doc_id}"`) < 0) {
-    throw new Error(`File not accounted for in sidebar ${filepath}`);
+  if (!accounted_for_in_sidebar(filepath)) {
+    throw new Error(
+      `File not accounted for in sidebar ${filepath} - try adding it to docs-website/sidebars.js`
+    );
   }
 }

docs-website/sidebars.js

@@ -1,3 +1,24 @@
+const fs = require("fs");
+
+function list_ids_in_directory(directory) {
+  const files = fs.readdirSync(`../${directory}`);
+  let ids = [];
+  for (const name of files) {
+    if (fs.lstatSync(`../${directory}/${name}`).isDirectory()) {
+      // Recurse into the directory.
+      const inner_ids = list_ids_in_directory(`${directory}/${name}`);
+      ids = ids.concat(inner_ids);
+    } else {
+      if (name.endsWith(".md")) {
+        const id = `${directory}/${name}`.replace(/\.md$/, "");
+        ids.push(id);
+      }
+    }
+  }
+  ids.sort();
+  return ids;
+}
+
 module.exports = {
   // users
   // architects
@@ -107,16 +128,7 @@ module.exports = {
       "docs/CODE_OF_CONDUCT",
       "docs/rfc",
       {
-        RFCs: [
-          "docs/rfc/active/1778-dashboards/README",
-          "docs/rfc/active/1812-ml_models/README",
-          "docs/rfc/active/1820-azkaban-flow-job/README",
-          "docs/rfc/active/1841-lineage/field_level_lineage",
-          "docs/rfc/active/business_glossary/README",
-          "docs/rfc/active/graph_ql_frontend/queries",
-          "docs/rfc/active/react-app/README",
-          "docs/rfc/active/tags/README",
-        ],
+        RFCs: list_ids_in_directory("docs/rfc/active"),
       },
     ],
   },

docs/how/search-over-new-field.md

@@ -321,7 +321,7 @@ private static final Set<String> CORP_USER_FACET_FIELDS = ImmutableSet.of("cours
 
 ### 3: Add field in the Person entity
 
-In [person-entity.ts](../../datahub-web/%40datahub/data-models/addon/entity/person/person-entity.ts), add your new property
+In [person-entity.ts](../../datahub-web/@datahub/data-models/addon/entity/person/person-entity.ts), add your new property
 
 ```ts
 @alias('entity.courses')

gms/README.md

@@ -17,11 +17,11 @@ However, if you only want to build `DataHub GMS` specifically:
 ```
 
 ## Dependencies
-Before starting `DataHub GMS`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../docker/kafka),  
+Before starting `DataHub GMS`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../docker/kafka-setup),  
 [Elasticsearch](../docker/elasticsearch) and [MySQL](../docker/mysql) Docker containers are up and running.
 
 ## Start via Docker image
-Quickest way to try out `DataHub GMS` is running the [Docker image](../docker/gms).
+Quickest way to try out `DataHub GMS` is running the [Docker image](../docker/datahub-gms).
 
 ## Start via command line
 If you do modify things and want to try it out quickly without building the Docker image, you can also run