diff --git a/docs-website/generateDocsDir.ts b/docs-website/generateDocsDir.ts index 4fa53a20d5..d571b65ea3 100644 --- a/docs-website/generateDocsDir.ts +++ b/docs-website/generateDocsDir.ts @@ -12,11 +12,30 @@ const GITHUB_BROWSE_URL = "https://github.com/linkedin/datahub/blob/master"; const SIDEBARS_DEF_PATH = "./sidebars.js"; const sidebars = require(SIDEBARS_DEF_PATH); +const sidebars_json = JSON.stringify(sidebars); +const sidebars_text = fs.readFileSync(SIDEBARS_DEF_PATH).toString(); +console.log(sidebars_json); function actually_in_sidebar(filepath: string): boolean { - const slug = get_slug(filepath); - const json = JSON.stringify(sidebars); - return json.indexOf(slug) < 0; + const doc_id = get_id(filepath); + return sidebars_json.indexOf(`"${doc_id}"`) >= 0; +} + +function accounted_for_in_sidebar(filepath: string): boolean { + if (actually_in_sidebar(filepath)) { + return true; + } + + // If we want to explicitly not include a doc in the sidebar but still have it + // available via a direct link, then keeping it commented out in the sidebar + // serves this purpose. To make it work, we search the text of the sidebar JS + // file. + const doc_id = get_id(filepath); + if (sidebars_text.indexOf(`"${doc_id}"`) >= 0) { + return true; + } + + return false; } function list_markdown_files(): string[] { @@ -81,6 +100,15 @@ const hardcoded_titles = { "docs/demo.md": "Demo", }; +// FIXME: Eventually, we'd like to fix all of the broken links within these files. +const allowed_broken_links = [ + "docs/architecture/metadata-serving.md", + "docs/developers.md", + "docs/how/customize-elasticsearch-query-template.md", + "docs/how/graph-onboarding.md", + "docs/how/search-onboarding.md", +]; + function markdown_guess_title( contents: matter.GrayMatterFile, filepath: string @@ -177,11 +205,16 @@ function new_url(original: string, filepath: string): string { // A reference to a file or directory in the Github repo. const relation = path.dirname(filepath); const updated_path = path.normalize(`${relation}/${original}`); - if (!fs.existsSync(`../${updated_path}`) && actually_in_sidebar(filepath)) { + const check_path = updated_path.replace(/#.+$/, ""); + if ( + !fs.existsSync(`../${check_path}`) && + actually_in_sidebar(filepath) && + !allowed_broken_links.includes(filepath) + ) { // Detects when the path is a dangling reference, according to the locally // checked out repo. throw new Error( - `broken github repo link: ${updated_path} in ${filepath}` + `broken github repo link to ${updated_path} in ${filepath}` ); } const updated_url = `${GITHUB_BROWSE_URL}/${updated_path}`; @@ -258,12 +291,12 @@ for (const filepath of markdown_files) { fs.writeFileSync(outpath, contents.stringify("")); } -// Output a list of all docs which are not included in a sidebar. +// Error if a doc is not accounted for in a sidebar. const sidebar = fs.readFileSync(SIDEBARS_DEF_PATH).toString(); for (const filepath of markdown_files) { - const doc_id = get_id(filepath); - - if (sidebar.indexOf(`"${doc_id}"`) < 0) { - throw new Error(`File not accounted for in sidebar ${filepath}`); + if (!accounted_for_in_sidebar(filepath)) { + throw new Error( + `File not accounted for in sidebar ${filepath} - try adding it to docs-website/sidebars.js` + ); } } diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index dd8628ed32..457cdb448c 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -1,3 +1,24 @@ +const fs = require("fs"); + +function list_ids_in_directory(directory) { + const files = fs.readdirSync(`../${directory}`); + let ids = []; + for (const name of files) { + if (fs.lstatSync(`../${directory}/${name}`).isDirectory()) { + // Recurse into the directory. + const inner_ids = list_ids_in_directory(`${directory}/${name}`); + ids = ids.concat(inner_ids); + } else { + if (name.endsWith(".md")) { + const id = `${directory}/${name}`.replace(/\.md$/, ""); + ids.push(id); + } + } + } + ids.sort(); + return ids; +} + module.exports = { // users // architects @@ -107,16 +128,7 @@ module.exports = { "docs/CODE_OF_CONDUCT", "docs/rfc", { - RFCs: [ - "docs/rfc/active/1778-dashboards/README", - "docs/rfc/active/1812-ml_models/README", - "docs/rfc/active/1820-azkaban-flow-job/README", - "docs/rfc/active/1841-lineage/field_level_lineage", - "docs/rfc/active/business_glossary/README", - "docs/rfc/active/graph_ql_frontend/queries", - "docs/rfc/active/react-app/README", - "docs/rfc/active/tags/README", - ], + RFCs: list_ids_in_directory("docs/rfc/active"), }, ], }, diff --git a/docs/how/search-over-new-field.md b/docs/how/search-over-new-field.md index 6485b2b5c7..2cbf919673 100644 --- a/docs/how/search-over-new-field.md +++ b/docs/how/search-over-new-field.md @@ -321,7 +321,7 @@ private static final Set CORP_USER_FACET_FIELDS = ImmutableSet.of("cours ### 3: Add field in the Person entity -In [person-entity.ts](../../datahub-web/%40datahub/data-models/addon/entity/person/person-entity.ts), add your new property +In [person-entity.ts](../../datahub-web/@datahub/data-models/addon/entity/person/person-entity.ts), add your new property ```ts @alias('entity.courses') diff --git a/gms/README.md b/gms/README.md index 3e30e05a50..e4c94c7c9d 100644 --- a/gms/README.md +++ b/gms/README.md @@ -17,11 +17,11 @@ However, if you only want to build `DataHub GMS` specifically: ``` ## Dependencies -Before starting `DataHub GMS`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../docker/kafka), +Before starting `DataHub GMS`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../docker/kafka-setup), [Elasticsearch](../docker/elasticsearch) and [MySQL](../docker/mysql) Docker containers are up and running. ## Start via Docker image -Quickest way to try out `DataHub GMS` is running the [Docker image](../docker/gms). +Quickest way to try out `DataHub GMS` is running the [Docker image](../docker/datahub-gms). ## Start via command line If you do modify things and want to try it out quickly without building the Docker image, you can also run