mirror of
https://github.com/datahub-project/datahub.git
synced 2025-06-27 05:03:31 +00:00
docs(ingest): add a guide for writing sources (#2575)
This commit is contained in:
Harshal Sheth
GitHub
parent
27c4607a56
commit
deca4a5073
@ -1,6 +1,5 @@
|
||||
---
|
||||
title: "Deploying with Kubernetes"
|
||||
hide_title: true
|
||||
---
|
||||
|
||||
# Deploying Datahub with Kubernetes
|
||||
|
||||
@ -122,6 +122,7 @@ function markdown_guess_title(
|
||||
filepath: string
|
||||
): void {
|
||||
if (contents.data.title) {
|
||||
contents.data.sidebar_label = contents.data.title;
|
||||
return;
|
||||
}
|
||||
|
||||
@ -138,13 +139,15 @@ function markdown_guess_title(
|
||||
throw new Error(`too many h1 headers in ${filepath}`);
|
||||
}
|
||||
title = headers[0].slice(2).trim();
|
||||
if (title.startsWith("DataHub ")) {
|
||||
title = title.slice(8).trim();
|
||||
}
|
||||
}
|
||||
|
||||
contents.data.title = title;
|
||||
contents.data.hide_title = true;
|
||||
|
||||
let sidebar_label = title;
|
||||
if (sidebar_label.startsWith("DataHub ")) {
|
||||
sidebar_label = sidebar_label.slice(8).trim();
|
||||
}
|
||||
contents.data.sidebar_label = sidebar_label;
|
||||
}
|
||||
|
||||
function markdown_add_edit_url(
|
||||
|
||||
@ -15,9 +15,9 @@
|
||||
"lint-check": "prettier -l generateDocsDir.ts sidebars.js src/pages/index.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@docusaurus/core": "^2.0.0-alpha.75",
|
||||
"@docusaurus/plugin-ideal-image": "^2.0.0-alpha.75",
|
||||
"@docusaurus/preset-classic": "^2.0.0-alpha.75",
|
||||
"@docusaurus/core": "^2.0.0-beta.0",
|
||||
"@docusaurus/plugin-ideal-image": "^2.0.0-beta.0",
|
||||
"@docusaurus/preset-classic": "^2.0.0-beta.0",
|
||||
"clsx": "^1.1.1",
|
||||
"react": "^16.12.0",
|
||||
"react-dom": "^16.12.0",
|
||||
|
||||
@ -67,7 +67,7 @@ module.exports = {
|
||||
// TODO: the titles of these should not be in question form in the sidebar
|
||||
"docs/developers",
|
||||
"docs/docker/development",
|
||||
"metadata-ingestion/README",
|
||||
"metadata-ingestion/adding-source",
|
||||
"docs/what/graph",
|
||||
"docs/what/search-index",
|
||||
"docs/how/add-new-aspect",
|
||||
|
||||
@ -1159,10 +1159,10 @@
|
||||
"@docsearch/css" "3.0.0-alpha.36"
|
||||
algoliasearch "^4.0.0"
|
||||
|
||||
"@docusaurus/core@2.0.0-alpha.75", "@docusaurus/core@^2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/core/-/core-2.0.0-alpha.75.tgz#96fa241d67e6975d0c599e73651f066624ccae68"
|
||||
integrity sha512-Ehds9MLwI6MxMb1FWBPZEocNDgByGFXDU5iptaNcFix6G1OInBi4fDZ4en2wtICx/5ahnSP4xfCPPG3CDsUlXw==
|
||||
"@docusaurus/core@2.0.0-beta.0", "@docusaurus/core@^2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/core/-/core-2.0.0-beta.0.tgz#05506ee02e7d40e9f4c8d7b4f918d26d3b191159"
|
||||
integrity sha512-xWwpuEwFRKJmZvNGOpr/dyRDnx/psckLPsozQTg2hu3u81Wqu9gigWgYK/C2fPlEjxMcVw0/2WH+zwpbyWmF2Q==
|
||||
dependencies:
|
||||
"@babel/core" "^7.12.16"
|
||||
"@babel/generator" "^7.12.15"
|
||||
@ -1174,11 +1174,11 @@
|
||||
"@babel/runtime" "^7.12.5"
|
||||
"@babel/runtime-corejs3" "^7.12.13"
|
||||
"@babel/traverse" "^7.12.13"
|
||||
"@docusaurus/cssnano-preset" "2.0.0-alpha.75"
|
||||
"@docusaurus/cssnano-preset" "2.0.0-beta.0"
|
||||
"@docusaurus/react-loadable" "5.5.0"
|
||||
"@docusaurus/types" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils-validation" "2.0.0-alpha.75"
|
||||
"@docusaurus/types" "2.0.0-beta.0"
|
||||
"@docusaurus/utils" "2.0.0-beta.0"
|
||||
"@docusaurus/utils-validation" "2.0.0-beta.0"
|
||||
"@endiliey/static-site-generator-webpack-plugin" "^4.0.0"
|
||||
"@svgr/webpack" "^5.5.0"
|
||||
autoprefixer "^10.2.5"
|
||||
@ -1241,34 +1241,34 @@
|
||||
webpack-merge "^5.7.3"
|
||||
webpackbar "^5.0.0-3"
|
||||
|
||||
"@docusaurus/cssnano-preset@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/cssnano-preset/-/cssnano-preset-2.0.0-alpha.75.tgz#864c806e114bd9babb7d1269818f68a6c3b45320"
|
||||
integrity sha512-r2wMpZGBNToyRCho5O1c0rmZ7fhW8PvCGzHLizTGfRtM+nFdsCFHxPRw3jxeGVEwd+kTT6Hz770sDom7J+Gzsw==
|
||||
"@docusaurus/cssnano-preset@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/cssnano-preset/-/cssnano-preset-2.0.0-beta.0.tgz#a79223479666059565d60a505bed2bbcac770384"
|
||||
integrity sha512-gqQHeQCDHZDd5NaiKZwDiyg75sBCqDyAsvmFukkDAty8xE7u9IhzbOQKvCAtwseuvzu2BNN41gnJ8bz7vZzQiw==
|
||||
dependencies:
|
||||
cssnano-preset-advanced "^5.0.0"
|
||||
postcss "^8.2.10"
|
||||
postcss-sort-media-queries "^3.8.9"
|
||||
|
||||
"@docusaurus/lqip-loader@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/lqip-loader/-/lqip-loader-2.0.0-alpha.75.tgz#b974b00b255f6d23dc3c8b80c3480de3ff0ce0dc"
|
||||
integrity sha512-Nmqn8CZNbl3M9YFvp86sWLn5satHJO3Ih/GaVEJavLsWCyzzyaeU5uE3ebgl3uFdMapb2JLzex21cdzFwT+NoQ==
|
||||
"@docusaurus/lqip-loader@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/lqip-loader/-/lqip-loader-2.0.0-beta.0.tgz#bc2064ee8328595ba3cafbb96eb1437a0ee491ac"
|
||||
integrity sha512-I6mrdKZQNrwFJ87TpTTBPY+m9Jt+tj7lUasMw/M6I//2BIzKB6+JO2hQXW5j+UUtSN/pGi1leOJgAiK1GE4dhQ==
|
||||
dependencies:
|
||||
file-loader "^6.2.0"
|
||||
lodash "^4.17.20"
|
||||
node-vibrant "^3.1.5"
|
||||
sharp "^0.27.1"
|
||||
|
||||
"@docusaurus/mdx-loader@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/mdx-loader/-/mdx-loader-2.0.0-alpha.75.tgz#37c62005c550afd0bee2b84c85ac597fd6f786b9"
|
||||
integrity sha512-ffGysExDvjoRaNvj7NBy8W5PaaprRKYUaybMAKtbCdGDq2Euawxeibk7MMYAElxCAGQKwf669HH0A91CLkl9Vw==
|
||||
"@docusaurus/mdx-loader@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/mdx-loader/-/mdx-loader-2.0.0-beta.0.tgz#7a58933994b2face62e34698db2f9c88c53c6d61"
|
||||
integrity sha512-oQLS2ZeUnqw79CV37glglZpaYgFfA5Az5lT83m5tJfMUZjoK4ehG1XWBeUzWy8QQNI452yAID8jz8jihEQeCcw==
|
||||
dependencies:
|
||||
"@babel/parser" "^7.12.16"
|
||||
"@babel/traverse" "^7.12.13"
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
"@docusaurus/utils" "2.0.0-beta.0"
|
||||
"@mdx-js/mdx" "^1.6.21"
|
||||
"@mdx-js/react" "^1.6.21"
|
||||
escape-html "^1.0.3"
|
||||
@ -1283,16 +1283,16 @@
|
||||
url-loader "^4.1.1"
|
||||
webpack "^5.28.0"
|
||||
|
||||
"@docusaurus/plugin-content-blog@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-blog/-/plugin-content-blog-2.0.0-alpha.75.tgz#a8e97c4aac1117c175318a69ca38ea97f2493653"
|
||||
integrity sha512-OqCR7+c5O0gXa5jBxOLio36fitEzJtiynEeTlF0KZm6X/SXXZm9m6eru6n2fSJW6YO9k4eDsmeBH9fiOnqPVIg==
|
||||
"@docusaurus/plugin-content-blog@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-blog/-/plugin-content-blog-2.0.0-beta.0.tgz#ea7d3679ab252e8f0e58aaf80f1fc6001c72c755"
|
||||
integrity sha512-lz63i5k/23RJ3Rk/2fIsYAoD8Wua3b5b0AbH2JoOhQu1iAIQiV8m91Z3XALBSzA3nBtAOIweNI7yzWL+JFSTvw==
|
||||
dependencies:
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/mdx-loader" "2.0.0-alpha.75"
|
||||
"@docusaurus/types" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils-validation" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
"@docusaurus/mdx-loader" "2.0.0-beta.0"
|
||||
"@docusaurus/types" "2.0.0-beta.0"
|
||||
"@docusaurus/utils" "2.0.0-beta.0"
|
||||
"@docusaurus/utils-validation" "2.0.0-beta.0"
|
||||
chalk "^4.1.0"
|
||||
feed "^4.2.2"
|
||||
fs-extra "^9.1.0"
|
||||
@ -1304,16 +1304,16 @@
|
||||
tslib "^2.1.0"
|
||||
webpack "^5.28.0"
|
||||
|
||||
"@docusaurus/plugin-content-docs@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-docs/-/plugin-content-docs-2.0.0-alpha.75.tgz#2aa04d7bf07e26c684be2d62ae1476a1de7538e9"
|
||||
integrity sha512-GEuyBBpLBJh4L/D9t3Othjj8FfiHx6roWFdXkYq5W8W/lQ8rq5wk7guu5nwN8l0zSLOrIk1+KZRnBEcwGGcZtQ==
|
||||
"@docusaurus/plugin-content-docs@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-docs/-/plugin-content-docs-2.0.0-beta.0.tgz#a5a1e0e95e499eefee53e4f61aeb99ac4a669648"
|
||||
integrity sha512-WdDQUh2rRCbfJswVc0vY9EaAspxgziqpVEZja8+BmQR/TZh7HuLplT6GJbiFbE4RvwM3+PwG/jHMPglYDK60kw==
|
||||
dependencies:
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/mdx-loader" "2.0.0-alpha.75"
|
||||
"@docusaurus/types" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils-validation" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
"@docusaurus/mdx-loader" "2.0.0-beta.0"
|
||||
"@docusaurus/types" "2.0.0-beta.0"
|
||||
"@docusaurus/utils" "2.0.0-beta.0"
|
||||
"@docusaurus/utils-validation" "2.0.0-beta.0"
|
||||
chalk "^4.1.0"
|
||||
combine-promises "^1.1.0"
|
||||
execa "^5.0.0"
|
||||
@ -1329,16 +1329,16 @@
|
||||
utility-types "^3.10.0"
|
||||
webpack "^5.28.0"
|
||||
|
||||
"@docusaurus/plugin-content-pages@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-pages/-/plugin-content-pages-2.0.0-alpha.75.tgz#4f4a6e041018990b82c914b8d5fe22791bbd68ba"
|
||||
integrity sha512-MISmA2BwMv9iMy7a8DGmWS6awUHjGhA3ulstud8QDwJrBJJINLN59kkZ2p7I36nN9Im3LNk2JIRwLZhALdDelQ==
|
||||
"@docusaurus/plugin-content-pages@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-pages/-/plugin-content-pages-2.0.0-beta.0.tgz#1cab3ebe0a08be74576f10c95675291bf84f848e"
|
||||
integrity sha512-mk5LVVSvn+HJPKBaAs/Pceq/hTGxF2LVBvJEquuQz0NMAW3QdBWaYRRpOrL9CO8v+ygn5RuLslXsyZBsDNuhww==
|
||||
dependencies:
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/mdx-loader" "2.0.0-alpha.75"
|
||||
"@docusaurus/types" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils-validation" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
"@docusaurus/mdx-loader" "2.0.0-beta.0"
|
||||
"@docusaurus/types" "2.0.0-beta.0"
|
||||
"@docusaurus/utils" "2.0.0-beta.0"
|
||||
"@docusaurus/utils-validation" "2.0.0-beta.0"
|
||||
globby "^11.0.2"
|
||||
lodash "^4.17.20"
|
||||
minimatch "^3.0.4"
|
||||
@ -1347,74 +1347,74 @@
|
||||
tslib "^2.1.0"
|
||||
webpack "^5.28.0"
|
||||
|
||||
"@docusaurus/plugin-debug@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-debug/-/plugin-debug-2.0.0-alpha.75.tgz#8e6ee18643d6e1fe392186d7c3010ed30be5f38b"
|
||||
integrity sha512-sXqxGq4RV8zIOUGMOn/EVhulY8SL6yco/aZXkOTwnJPv9zwe0L0d3/hl4BYpOygfJmnM1YiOlaXzlz/JLgRzxg==
|
||||
"@docusaurus/plugin-debug@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-debug/-/plugin-debug-2.0.0-beta.0.tgz#bee672b8858d88bdb229d4301785ff4692ebd17f"
|
||||
integrity sha512-m75sZdF8Yccxfih3qfdQg9DucMTrYBnmeTA8GNmdVaK701Ip8t50d1pDJchtu0FSEh6vzVB9C6D2YD5YgVFp8A==
|
||||
dependencies:
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/types" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
"@docusaurus/types" "2.0.0-beta.0"
|
||||
"@docusaurus/utils" "2.0.0-beta.0"
|
||||
react-json-view "^1.21.1"
|
||||
tslib "^2.1.0"
|
||||
|
||||
"@docusaurus/plugin-google-analytics@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-analytics/-/plugin-google-analytics-2.0.0-alpha.75.tgz#2ba02a9f003280358506c02da5400c08cf966db0"
|
||||
integrity sha512-doK2UU3ox7IvyTkfEkjzge/t89EhynDwhmhY3dWGxGVMrWA3PuzelnISHBrOthuiJYFcUIwCzUaObvYBqKA0wA==
|
||||
"@docusaurus/plugin-google-analytics@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-analytics/-/plugin-google-analytics-2.0.0-beta.0.tgz#ee287fb991202d8e9b792129dcc5542ef3ccd6c9"
|
||||
integrity sha512-7lHrg1L+adc8VbiaLexa15i4fdq4MRPUTLMxRPAWz+QskhisW89Ryi2/gDmfMNqLblX84Qg2RASa+2gqO4wepw==
|
||||
dependencies:
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
|
||||
"@docusaurus/plugin-google-gtag@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-gtag/-/plugin-google-gtag-2.0.0-alpha.75.tgz#837ab0593b77956236c5f009111f42f373e5ff24"
|
||||
integrity sha512-VQSqdvC3LC4JcQhKEQfFDoYrz+66aVxbgJNrmHHK/J8+Iha1KoMNnGctFw58RjT1jw2R45lajlY3TLrSiZsS7w==
|
||||
"@docusaurus/plugin-google-gtag@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-gtag/-/plugin-google-gtag-2.0.0-beta.0.tgz#4836770130cf54ff2cd83affbff9644ee7293e9e"
|
||||
integrity sha512-V7zaYbhAMv0jexm5H/5sAnoM1GHibcn9QQk5UWC++x1kE0KRuLDZHV+9OyvW5wr0wWFajod/b88SpUpSMF5u+g==
|
||||
dependencies:
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
|
||||
"@docusaurus/plugin-ideal-image@^2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-ideal-image/-/plugin-ideal-image-2.0.0-alpha.75.tgz#287bcad80c4cf3173aaecd4768c6baf105efc07c"
|
||||
integrity sha512-dliSgDNFvbp2GLS6TtidNxzQRtsUTTi0K0b32BP/RPngftPud4WtruFebvJ8vyYHYWMP/+jRZfeUA8MxPoCMrg==
|
||||
"@docusaurus/plugin-ideal-image@^2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-ideal-image/-/plugin-ideal-image-2.0.0-beta.0.tgz#46b2eae9ef16fa24f91cdc346405b5feb27eaada"
|
||||
integrity sha512-Bj5kIpetwViZoGFZlPZnq7BUa7ABuZfzzP68dUoiJ21ieeGanilAjEHym4aXgGBUfSFoyYtkakx29neRFAf+eQ==
|
||||
dependencies:
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/lqip-loader" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
"@docusaurus/lqip-loader" "2.0.0-beta.0"
|
||||
"@docusaurus/responsive-loader" "1.4.0"
|
||||
"@docusaurus/types" "2.0.0-alpha.75"
|
||||
"@docusaurus/types" "2.0.0-beta.0"
|
||||
"@endiliey/react-ideal-image" "^0.0.11"
|
||||
react-waypoint "^9.0.2"
|
||||
sharp "^0.27.1"
|
||||
tslib "^2.1.0"
|
||||
webpack "^5.28.0"
|
||||
|
||||
"@docusaurus/plugin-sitemap@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-sitemap/-/plugin-sitemap-2.0.0-alpha.75.tgz#b3d5ea20ba9b78d09dfc4ba504bbde2939863ca5"
|
||||
integrity sha512-v6kAfkg10U5x2uxXvZnK9imiP56mUXu2TtGOV2eUeGy1ZeclszLZ/9xFyQcWqXheEu+KjvCM0OO2R3q017PnxQ==
|
||||
"@docusaurus/plugin-sitemap@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-sitemap/-/plugin-sitemap-2.0.0-beta.0.tgz#985d4cc3af86499f616ced024ba1fab8329e601d"
|
||||
integrity sha512-dvmk8Sr+6pBkiKDb7Rjdp0GeFDWPUlayoJWK3fN3g0Fno6uxFfYhNZyXJ+ObyCA7HoW5rzeBMiO+uAja19JXTg==
|
||||
dependencies:
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/types" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils-validation" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
"@docusaurus/types" "2.0.0-beta.0"
|
||||
"@docusaurus/utils" "2.0.0-beta.0"
|
||||
"@docusaurus/utils-validation" "2.0.0-beta.0"
|
||||
fs-extra "^9.1.0"
|
||||
sitemap "^6.3.6"
|
||||
tslib "^2.1.0"
|
||||
|
||||
"@docusaurus/preset-classic@^2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/preset-classic/-/preset-classic-2.0.0-alpha.75.tgz#315aab488504e5a702a6cb2c81949fecf88fa845"
|
||||
integrity sha512-xafacmwkW9MVFCaB8s7L2IRSV9GsPCoD2lcfAEyMSA+6NIXUQToSz50jbJ49/goK4NZCYXgFqoBBr6vecFfM2w==
|
||||
"@docusaurus/preset-classic@^2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/preset-classic/-/preset-classic-2.0.0-beta.0.tgz#79eb4366e6b5eb7061370019127e40172432d770"
|
||||
integrity sha512-cFpR0UaAeUt5qVx1bpidhlar6tiRNITIQlxP4bOVsjbxVTZhZ/cNuIz7C+2zFPCuKIflGXdTIQOrucPmd7z51Q==
|
||||
dependencies:
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-content-blog" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-content-docs" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-content-pages" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-debug" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-google-analytics" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-google-gtag" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-sitemap" "2.0.0-alpha.75"
|
||||
"@docusaurus/theme-classic" "2.0.0-alpha.75"
|
||||
"@docusaurus/theme-search-algolia" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-content-blog" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-content-docs" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-content-pages" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-debug" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-google-analytics" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-google-gtag" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-sitemap" "2.0.0-beta.0"
|
||||
"@docusaurus/theme-classic" "2.0.0-beta.0"
|
||||
"@docusaurus/theme-search-algolia" "2.0.0-beta.0"
|
||||
|
||||
"@docusaurus/react-loadable@5.5.0":
|
||||
version "5.5.0"
|
||||
@ -1430,19 +1430,19 @@
|
||||
dependencies:
|
||||
loader-utils "^2.0.0"
|
||||
|
||||
"@docusaurus/theme-classic@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/theme-classic/-/theme-classic-2.0.0-alpha.75.tgz#4c3720c1e8e1fe8081c1211418854fc4b7f4c01d"
|
||||
integrity sha512-yjZxiyBZtqI5FPIpFFdGc7lilA8+OMeCIpTnWy6DdgR5zgiIYB2xom1EoTLEk2jBBp6E2w5BR7nY4Lpro9efbw==
|
||||
"@docusaurus/theme-classic@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/theme-classic/-/theme-classic-2.0.0-beta.0.tgz#0ad74264dc592590bd7d8a6f6327cb83bbabc665"
|
||||
integrity sha512-cBNtwAyg3be7Gk41FazMtgyibAcfuYaGHhGHIDRsXfc/qp3RhbiGiei7tyh200QT0NgKZxiVQy/r4d0mtjC++Q==
|
||||
dependencies:
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-content-blog" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-content-docs" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-content-pages" "2.0.0-alpha.75"
|
||||
"@docusaurus/theme-common" "2.0.0-alpha.75"
|
||||
"@docusaurus/types" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils-validation" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-content-blog" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-content-docs" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-content-pages" "2.0.0-beta.0"
|
||||
"@docusaurus/theme-common" "2.0.0-beta.0"
|
||||
"@docusaurus/types" "2.0.0-beta.0"
|
||||
"@docusaurus/utils" "2.0.0-beta.0"
|
||||
"@docusaurus/utils-validation" "2.0.0-beta.0"
|
||||
"@mdx-js/mdx" "^1.6.21"
|
||||
"@mdx-js/react" "^1.6.21"
|
||||
chalk "^4.1.0"
|
||||
@ -1460,38 +1460,38 @@
|
||||
react-router-dom "^5.2.0"
|
||||
rtlcss "^3.1.2"
|
||||
|
||||
"@docusaurus/theme-common@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/theme-common/-/theme-common-2.0.0-alpha.75.tgz#840bc6616fd9fb6ee5107b7c7a189d16b73021a1"
|
||||
integrity sha512-RCKPWixJkfpmuchUImE3EYlW1ShPjn9UxK9SaeCtGFw7bAdKazEj99ikOYwlE+tPwbN7kiQQ1VJmz76Ts0F8sg==
|
||||
"@docusaurus/theme-common@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/theme-common/-/theme-common-2.0.0-beta.0.tgz#3674ef6482cc39efa034fd8d8b1c831588896329"
|
||||
integrity sha512-2rcVmQpvbdAgnzTWuM7Bfpu+2TQm928bhlvxn226jQy7IYz8ySRlIode63HhCtpx03hpdMCkrK6HxhfEcvHjQg==
|
||||
dependencies:
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-content-blog" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-content-docs" "2.0.0-alpha.75"
|
||||
"@docusaurus/plugin-content-pages" "2.0.0-alpha.75"
|
||||
"@docusaurus/types" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-content-blog" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-content-docs" "2.0.0-beta.0"
|
||||
"@docusaurus/plugin-content-pages" "2.0.0-beta.0"
|
||||
"@docusaurus/types" "2.0.0-beta.0"
|
||||
tslib "^2.1.0"
|
||||
|
||||
"@docusaurus/theme-search-algolia@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/theme-search-algolia/-/theme-search-algolia-2.0.0-alpha.75.tgz#6040cae3a89d0d9f0d0bf2c3afddd11cdd92b6c8"
|
||||
integrity sha512-AvAuMAqLomD27kJ5+r1tE2dD3tbOfF/ijYl2e49czBUdBgOToSUpabZzLMLyuLdktvf1TmylMCD2AoTpyEdCDw==
|
||||
"@docusaurus/theme-search-algolia@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/theme-search-algolia/-/theme-search-algolia-2.0.0-beta.0.tgz#bfdee3981d8da72377b9045459950686d28a01fd"
|
||||
integrity sha512-/GhgAm4yuwqTXWTsWnqpFYxpjTv+t45Wk8q/LmTVINa+A7b6jkMkch2lygagIt69/ufDm2Uw6eYhgrmF4DJqfQ==
|
||||
dependencies:
|
||||
"@docsearch/react" "^3.0.0-alpha.33"
|
||||
"@docusaurus/core" "2.0.0-alpha.75"
|
||||
"@docusaurus/theme-common" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils-validation" "2.0.0-alpha.75"
|
||||
"@docusaurus/core" "2.0.0-beta.0"
|
||||
"@docusaurus/theme-common" "2.0.0-beta.0"
|
||||
"@docusaurus/utils" "2.0.0-beta.0"
|
||||
"@docusaurus/utils-validation" "2.0.0-beta.0"
|
||||
algoliasearch "^4.8.4"
|
||||
algoliasearch-helper "^3.3.4"
|
||||
clsx "^1.1.1"
|
||||
eta "^1.12.1"
|
||||
lodash "^4.17.20"
|
||||
|
||||
"@docusaurus/types@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/types/-/types-2.0.0-alpha.75.tgz#a6a14d79cca914953a70429b9c503f7cff77c35b"
|
||||
integrity sha512-nid0POV6uVlfq12mwdPWcbeB5qzBJgrdOiL9tTijnJPHT7nCvAMtff4UEF249QSF2s47yDSxRY8jXWjMFjknNQ==
|
||||
"@docusaurus/types@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/types/-/types-2.0.0-beta.0.tgz#f473f417bdf690cfd52611ddf6d89ff939d1f2a4"
|
||||
integrity sha512-z9PI+GbtYwqTXnkX4/a/A6psDX2p8N2uWlN2f4ifrm8WY4WhR9yiTOh0uo0pIqqaUQQvkEq3o5hOXuXLECEs+w==
|
||||
dependencies:
|
||||
commander "^5.1.0"
|
||||
joi "^17.4.0"
|
||||
@ -1499,22 +1499,22 @@
|
||||
webpack "^5.28.0"
|
||||
webpack-merge "^5.7.3"
|
||||
|
||||
"@docusaurus/utils-validation@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/utils-validation/-/utils-validation-2.0.0-alpha.75.tgz#ec173cb7f68866915dda7d7145a48b7de256aab1"
|
||||
integrity sha512-0GxvDPz9BUUb1wESw8SNZ6jf7GSYYBRGOKH/Ja0XVfKmH1TC4yRJdncIOs0w8YUp1bJmoNdTpn4kI4NLAjLdwQ==
|
||||
"@docusaurus/utils-validation@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/utils-validation/-/utils-validation-2.0.0-beta.0.tgz#3e6491c269a397fed29717a1cb69109df9483461"
|
||||
integrity sha512-ELl/FVJ6xBz35TisZ1NmJhjbiVXDeU++K531PEFPCPmwnQPh7S6hZXdPnR71/Kc3BmuN9X2ZkwGOqNKVfys2Bg==
|
||||
dependencies:
|
||||
"@docusaurus/utils" "2.0.0-alpha.75"
|
||||
"@docusaurus/utils" "2.0.0-beta.0"
|
||||
chalk "^4.1.0"
|
||||
joi "^17.4.0"
|
||||
tslib "^2.1.0"
|
||||
|
||||
"@docusaurus/utils@2.0.0-alpha.75":
|
||||
version "2.0.0-alpha.75"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/utils/-/utils-2.0.0-alpha.75.tgz#f1f7b91f670b23db6750abbd589aaa283821f761"
|
||||
integrity sha512-rIMbwQce694jkgx/yoF53bLXbL3rjdlmDoOHQiiNTHERQ0I4duTWAVa5EXxjmK85IiqtH4Fn0+v8NybynhWqqQ==
|
||||
"@docusaurus/utils@2.0.0-beta.0":
|
||||
version "2.0.0-beta.0"
|
||||
resolved "https://registry.yarnpkg.com/@docusaurus/utils/-/utils-2.0.0-beta.0.tgz#6f2690fd6fcd942f0d690db1dffb96742762deb3"
|
||||
integrity sha512-bvrT1EQu0maavr0Hb/lke9jmpzgVL/9tn5VQtbyahf472eJFY0bQDExllDrHK+l784SUvucqX0iaQeg0q6ySUw==
|
||||
dependencies:
|
||||
"@docusaurus/types" "2.0.0-alpha.75"
|
||||
"@docusaurus/types" "2.0.0-beta.0"
|
||||
"@types/github-slugger" "^1.3.0"
|
||||
chalk "^4.1.0"
|
||||
escape-string-regexp "^4.0.0"
|
||||
|
||||
@ -1,4 +1,9 @@
|
||||
# How to model metadata ?
|
||||
---
|
||||
title: "Metadata Modeling"
|
||||
---
|
||||
|
||||
# How to model metadata?
|
||||
|
||||
[GMA](../what/gma.md) uses [rest.li](https://rest.li), which is LinkedIn's open source REST framework. All metadata in GMA needs to be modelled using [Pegasus schema (PDL)](https://linkedin.github.io/rest.li/pdl_schema) which is the data schema for [rest.li](https://rest.li).
|
||||
|
||||
Conceptually we’re modelling metadata as a hybrid graph of nodes ([entities](../what/entity.md)) and edges ([relationships](../what/relationship.md)), with additional documents ([metadata aspects](../what/aspect.md)) attached to each node. You can also think of it as a modified [Entity-Relationship Model](https://en.wikipedia.org/wiki/Entity%E2%80%93relationship_model).
|
||||
|
||||
@ -814,4 +814,4 @@ In order to use this example, you must first configure the Datahub hook. Like in
|
||||
|
||||
## Developing
|
||||
|
||||
See the [developing guide](./developing.md).
|
||||
See the [developing guide](./developing.md) or the [adding a source guide](./adding-source.md).
|
||||
|
||||
37
metadata-ingestion/adding-source.md
Normal file
37
metadata-ingestion/adding-source.md
Normal file
@ -0,0 +1,37 @@
|
||||
# Adding a Metadata Ingestion Source
|
||||
|
||||
:::note
|
||||
|
||||
This guide assumes that you've already followed the metadata ingestion [developing guide](./developing.md) to set up your local environment.
|
||||
|
||||
:::
|
||||
|
||||
### 1. Set up the configuration model
|
||||
|
||||
We use [pydantic](https://pydantic-docs.helpmanual.io/) for configuration, and all models must inherit from `ConfigModel`. The [file source](./src/datahub/ingestion/source/mce_file.py) is a good example.
|
||||
|
||||
### 2. Set up the reporter
|
||||
|
||||
The reporter interface enables the source to report statistics, warnings, failures, and other information about the run. Some sources use the default `SourceReport` class, but others inherit and extend that class.
|
||||
|
||||
### 3. Implement the source itself
|
||||
|
||||
The core for the source is the `get_workunits` method, which produces a stream of MCE objects. The [file source](./src/datahub/ingestion/source/mce_file.py) is a good and simple example.
|
||||
|
||||
The MetadataChangeEventClass is defined in the [metadata models](./src/datahub/metadata/schema_classes.py). There are also some [convenience methods](./src/datahub/emitter/mce_builder.py) for commonly used operations.
|
||||
|
||||
### 4. Set up the dependencies
|
||||
|
||||
Declare the source's pip dependencies in the `plugins` variable of the [setup script](./setup.py).
|
||||
|
||||
### 5. Enable discoverability
|
||||
|
||||
Declare the source under the `entry_points` variable of the [setup script](./setup.py). This enables the source to be listed when running `datahub check plugins`, and sets up the source's shortened alias for use in recipes.
|
||||
|
||||
### 6. Write tests
|
||||
|
||||
Tests go in the `tests` directory. We use the [pytest framework](https://pytest.org/).
|
||||
|
||||
### 7. Write docs
|
||||
|
||||
Add the plugin to the table at the top of the README file, and add the source's documentation underneath the sources header.
|
||||
@ -83,6 +83,8 @@ The syntax for installing plugins is slightly different in development. For exam
|
||||
|
||||
Contributions welcome!
|
||||
|
||||
Also take a look at the guide to [adding a source](./adding-source.md).
|
||||
|
||||
### Testing
|
||||
|
||||
```shell
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user