diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js index 29ce7269a4..ce9db79bfa 100644 --- a/docs-website/docusaurus.config.js +++ b/docs-website/docusaurus.config.js @@ -109,10 +109,6 @@ module.exports = { label: "Features", to: "docs/features", }, - { - label: "FAQs", - to: "docs/faq", - }, ], }, { @@ -189,6 +185,7 @@ module.exports = { path: "genDocs", sidebarPath: require.resolve("./sidebars.js"), editUrl: "https://github.com/linkedin/datahub/blob/master/", + numberPrefixParser: false, // TODO: make these work correctly with the doc generation showLastUpdateAuthor: true, showLastUpdateTime: true, diff --git a/docs-website/generateDocsDir.ts b/docs-website/generateDocsDir.ts index b976859fac..a3ee47d0c6 100644 --- a/docs-website/generateDocsDir.ts +++ b/docs-website/generateDocsDir.ts @@ -58,10 +58,25 @@ function accounted_for_in_sidebar(filepath: string): boolean { } function list_markdown_files(): string[] { - const all_markdown_files = execSync("cd .. && git ls-files . | grep '.md$'") + let all_markdown_files = execSync("git ls-files --full-name .. | grep '.md$'") .toString() .trim() .split("\n"); + if (!process.env.CI) { + // If not in CI, we also include "untracked" files. + const untracked_files = execSync( + "(git ls-files --full-name --others --exclude-standard .. | grep '.md$') || true" + ) + .toString() + .trim() + .split("\n") + .filter((filepath) => filepath); + + if (untracked_files.length > 0) { + console.log(`Including untracked files in docs list: ${untracked_files}`); + all_markdown_files = [...all_markdown_files, ...untracked_files]; + } + } const filter_patterns = [ // We don't need our issue and pull request templates. @@ -70,9 +85,9 @@ function list_markdown_files(): string[] { /^docs-website\//, // Don't want hosted docs for these. /^contrib\//, - // Keep main docs for kubernetes, but skip the inner docs + // Keep main docs for kubernetes, but skip the inner docs. /^datahub-kubernetes\//, - /^datahub-web\//, + // Various other docs/directories to ignore. /^metadata-ingestion-examples\//, /^docker\/(?!README|datahub-upgrade|airflow\/local_airflow)/, // Drop all but a few docker docs. /^docs\/rfc\/templates\/000-template\.md$/, @@ -101,6 +116,10 @@ const hardcoded_slugs = { }; function get_slug(filepath: string): string { + // The slug is the URL path to the page. + // In the actual site, all slugs are prefixed with /docs. + // There's no need to do this cleanup, but it does make the URLs a bit more aesthetic. + if (filepath in hardcoded_slugs) { return hardcoded_slugs[filepath]; } @@ -256,6 +275,7 @@ function new_url(original: string, filepath: string): string { ".sh", ".env", ".sql", + // Using startsWith since some URLs will be .ext#LINENO ].some((ext) => suffix.startsWith(ext)) ) { // A reference to a file or directory in the Github repo. @@ -342,6 +362,12 @@ function markdown_sanitize_and_linkify(content: string): string { "[#$1](https://github.com/linkedin/datahub/pull/$1)" ); + // Prettify bare links to PRs. + content = content.replace( + /(\s+)(https:\/\/github\.com\/linkedin\/datahub\/pull\/(\d+))(\s+|$)/g, + "$1[#$3]($2)$4" + ); + return content; } @@ -364,7 +390,9 @@ slug: /releases custom_edit_url: https://github.com/linkedin/datahub/blob/master/docs-website/generateDocsDir.ts --- -# DataHub Releases\n\n`); +# DataHub Releases + +## Summary\n\n`); const releases_list = await octokit.rest.repos.listReleases({ owner: "linkedin", @@ -464,7 +492,7 @@ function write_markdown_file( } // Error if a doc is not accounted for in a sidebar. - const autogenerated_sidebar_directories = ["docs/rfc/active/"]; + const autogenerated_sidebar_directories = []; for (const filepath of markdown_files) { if ( autogenerated_sidebar_directories.some((dir) => filepath.startsWith(dir)) diff --git a/docs-website/package.json b/docs-website/package.json index 7a9675eb16..932ed33833 100644 --- a/docs-website/package.json +++ b/docs-website/package.json @@ -46,4 +46,4 @@ "ts-node": "^9.1.1", "typescript": "^4.1.5" } -} \ No newline at end of file +} diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 069514ee0a..2c96e1c2e0 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -15,10 +15,7 @@ function list_ids_in_directory(directory, hardcoded_labels) { } else { if (name.endsWith(".md")) { const slug = name.replace(/\.md$/, ""); - let id = `${directory}/${slug}`; - if (id.match(/\/\d+-.+/)) { - id = id.replace(/\/\d+-/, "/"); - } + const id = `${directory}/${slug}`; if (id in hardcoded_labels) { label = hardcoded_labels[id];