chore(docs): various cleanup for docs-website (#4143)

docs-website/docusaurus.config.js

@@ -109,10 +109,6 @@ module.exports = {
               label: "Features",
               to: "docs/features",
             },
-            {
-              label: "FAQs",
-              to: "docs/faq",
-            },
           ],
         },
         {
@@ -189,6 +185,7 @@ module.exports = {
           path: "genDocs",
           sidebarPath: require.resolve("./sidebars.js"),
           editUrl: "https://github.com/linkedin/datahub/blob/master/",
+          numberPrefixParser: false,
           // TODO: make these work correctly with the doc generation
           showLastUpdateAuthor: true,
           showLastUpdateTime: true,

docs-website/generateDocsDir.ts

@@ -58,10 +58,25 @@ function accounted_for_in_sidebar(filepath: string): boolean {
 }
 
 function list_markdown_files(): string[] {
-  const all_markdown_files = execSync("cd .. && git ls-files . | grep '.md$'")
+  let all_markdown_files = execSync("git ls-files --full-name .. | grep '.md$'")
     .toString()
     .trim()
     .split("\n");
+  if (!process.env.CI) {
+    // If not in CI, we also include "untracked" files.
+    const untracked_files = execSync(
+      "(git ls-files --full-name --others --exclude-standard .. | grep '.md$') || true"
+    )
+      .toString()
+      .trim()
+      .split("\n")
+      .filter((filepath) => filepath);
+
+    if (untracked_files.length > 0) {
+      console.log(`Including untracked files in docs list: ${untracked_files}`);
+      all_markdown_files = [...all_markdown_files, ...untracked_files];
+    }
+  }
 
   const filter_patterns = [
     // We don't need our issue and pull request templates.
@@ -70,9 +85,9 @@ function list_markdown_files(): string[] {
     /^docs-website\//,
     // Don't want hosted docs for these.
     /^contrib\//,
-    // Keep main docs for kubernetes, but skip the inner docs
+    // Keep main docs for kubernetes, but skip the inner docs.
     /^datahub-kubernetes\//,
-    /^datahub-web\//,
+    // Various other docs/directories to ignore.
     /^metadata-ingestion-examples\//,
     /^docker\/(?!README|datahub-upgrade|airflow\/local_airflow)/, // Drop all but a few docker docs.
     /^docs\/rfc\/templates\/000-template\.md$/,
@@ -101,6 +116,10 @@ const hardcoded_slugs = {
 };
 
 function get_slug(filepath: string): string {
+  // The slug is the URL path to the page.
+  // In the actual site, all slugs are prefixed with /docs.
+  // There's no need to do this cleanup, but it does make the URLs a bit more aesthetic.
+
   if (filepath in hardcoded_slugs) {
     return hardcoded_slugs[filepath];
   }
@@ -256,6 +275,7 @@ function new_url(original: string, filepath: string): string {
       ".sh",
       ".env",
       ".sql",
+      // Using startsWith since some URLs will be .ext#LINENO
     ].some((ext) => suffix.startsWith(ext))
   ) {
     // A reference to a file or directory in the Github repo.
@@ -342,6 +362,12 @@ function markdown_sanitize_and_linkify(content: string): string {
     "[#$1](https://github.com/linkedin/datahub/pull/$1)"
   );
 
+  // Prettify bare links to PRs.
+  content = content.replace(
+    /(\s+)(https:\/\/github\.com\/linkedin\/datahub\/pull\/(\d+))(\s+|$)/g,
+    "$1[#$3]($2)$4"
+  );
+
   return content;
 }
 
@@ -364,7 +390,9 @@ slug: /releases
 custom_edit_url: https://github.com/linkedin/datahub/blob/master/docs-website/generateDocsDir.ts
 ---
 
-# DataHub Releases\n\n`);
+# DataHub Releases
+
+## Summary\n\n`);
 
   const releases_list = await octokit.rest.repos.listReleases({
     owner: "linkedin",
@@ -464,7 +492,7 @@ function write_markdown_file(
   }
 
   // Error if a doc is not accounted for in a sidebar.
-  const autogenerated_sidebar_directories = ["docs/rfc/active/"];
+  const autogenerated_sidebar_directories = [];
   for (const filepath of markdown_files) {
     if (
       autogenerated_sidebar_directories.some((dir) => filepath.startsWith(dir))

docs-website/sidebars.js

@@ -15,10 +15,7 @@ function list_ids_in_directory(directory, hardcoded_labels) {
     } else {
       if (name.endsWith(".md")) {
         const slug = name.replace(/\.md$/, "");
-        let id = `${directory}/${slug}`;
-        if (id.match(/\/\d+-.+/)) {
-          id = id.replace(/\/\d+-/, "/");
-        }
+        const id = `${directory}/${slug}`;
 
         if (id in hardcoded_labels) {
           label = hardcoded_labels[id];