mirror of
https://github.com/datahub-project/datahub.git
synced 2025-07-18 23:02:11 +00:00
189 lines
114 KiB
HTML
189 lines
114 KiB
HTML
<!doctype html>
|
||
<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-docs/modeling/extending-the-metadata-model" data-has-hydrated="false">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="generator" content="Docusaurus v2.4.3">
|
||
<title data-rh="true">Extending the Metadata Model | DataHub</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://docs.datahub.com/docs/metadata-modeling/extending-the-metadata-model"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Extending the Metadata Model | DataHub"><meta data-rh="true" name="description" content="You can extend the metadata model by either creating a new Entity or extending an existing one. Unsure if you need to"><meta data-rh="true" property="og:description" content="You can extend the metadata model by either creating a new Entity or extending an existing one. Unsure if you need to"><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://docs.datahub.com/docs/metadata-modeling/extending-the-metadata-model"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/metadata-modeling/extending-the-metadata-model" hreflang="en"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/metadata-modeling/extending-the-metadata-model" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://RK0UG797F3-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/learn/rss.xml" title="DataHub RSS Feed">
|
||
<link rel="alternate" type="application/atom+xml" href="/learn/atom.xml" title="DataHub Atom Feed">
|
||
|
||
<link rel="preconnect" href="https://www.google-analytics.com">
|
||
<link rel="preconnect" href="https://www.googletagmanager.com">
|
||
<script async src="https://www.googletagmanager.com/gtag/js?id=G-PKGVLETT4C"></script>
|
||
<script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-PKGVLETT4C",{})</script>
|
||
<link rel="preconnect" href="https://www.googletagmanager.com">
|
||
<script>window.dataLayer=window.dataLayer||[]</script>
|
||
<script>!function(e,t,a,n,g){e[n]=e[n]||[],e[n].push({"gtm.start":(new Date).getTime(),event:"gtm.js"});var m=t.getElementsByTagName(a)[0],r=t.createElement(a);r.async=!0,r.src="https://www.googletagmanager.com/gtm.js?id=GTM-5M8T9HNN",m.parentNode.insertBefore(r,m)}(window,document,"script","dataLayer")</script>
|
||
|
||
|
||
<link rel="search" type="application/opensearchdescription+xml" title="DataHub" href="/opensearch.xml">
|
||
|
||
|
||
|
||
|
||
<meta httpequiv="Content-Security-Policy" content="frame-ancestors 'self' https://*.acryl.io https://acryldata.io http://localhost:*">
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap">
|
||
<script src="https://tools.luckyorange.com/core/lo.js?site-id=28ea8a38" async defer="defer"></script>
|
||
<script src="/scripts/rb2b.js" async defer="defer"></script>
|
||
<script src="https://app.revenuehero.io/scheduler.min.js"></script>
|
||
<script src="https://tag.clearbitscripts.com/v1/pk_2e321cabe30432a5c44c0424781aa35f/tags.js" referrerpolicy="strict-origin-when-cross-origin"></script>
|
||
<script src="/scripts/reo.js"></script>
|
||
<script id="runllm-widget-script" type="module" src="https://widget.runllm.com" crossorigin="true" runllm-name="DataHub" runllm-assistant-id="81" runllm-position="BOTTOM_RIGHT" runllm-keyboard-shortcut="Mod+j" runllm-preset="docusaurus" runllm-theme-color="#1890FF" runllm-brand-logo="https://docs.datahub.com/img/datahub-logo-color-mark.svg" runllm-community-url="https://datahub.com/slack" runllm-community-type="slack" runllm-disable-ask-a-person="true" async></script><link rel="stylesheet" href="/assets/css/styles.d8fe2eb8.css">
|
||
<link rel="preload" href="/assets/js/runtime~main.50e13f51.js" as="script">
|
||
<link rel="preload" href="/assets/js/main.edc0853c.js" as="script">
|
||
</head>
|
||
<body class="navigation-with-keyboard">
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-5M8T9HNN" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
|
||
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
|
||
<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:transparent;color:#ffffff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"><div class="shimmer-banner"><p>DataHub Secures $35 Million Series B</p><a href="https://datahub.com/news/series-b-announcement/" target="_blank" class="button"><div>Read the announcement<span> →</span></div></a></div></div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a href="https://datahub.com" target="_blank" rel="noopener noreferrer" class="navbar__brand"><div class="navbar__logo"><img src="/img/datahub-logo-color-light-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/datahub-logo-color-dark-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link versionNavItem_cbn8">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/metadata-modeling/extending-the-metadata-model">Next</a></li><li><a class="dropdown__link" href="/docs/1.1.0/metadata-modeling/extending-the-metadata-model">1.1.0</a></li><li><hr class="dropdown-separator" style="margin: 0.4rem;"></li><li><div class="dropdown__link"><b>Archived versions</b></div></li><li>
|
||
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/features">1.0.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/0.15.0/features">0.15.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-8jkm4uler-acryldata.vercel.app/docs/0.14.1/features">0.14.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-eue2qafvn-acryldata.vercel.app/docs/features">0.14.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-psat3nzgi-acryldata.vercel.app/docs/features">0.13.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-lzxh86531-acryldata.vercel.app/docs/features">0.13.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-2uuxmgza2-acryldata.vercel.app/docs/features">0.12.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-irpoe2osc-acryldata.vercel.app/docs/features">0.11.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-1gv2yzn9d-acryldata.vercel.app/docs/features">0.10.5
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li></ul></div></div><div class="navbar__items navbar__items--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/docs">Docs</a><a class="navbar__item navbar__link" href="/integrations">Integrations</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/weekly-demo" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Weekly Demo"></div><div class="title_c7DP">Weekly Demo</div></a></div><div><a href="https://datahub.com/use-cases" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-forum.png" alt="Use Cases"></div><div class="title_c7DP">Use Cases</div></a></div><div><a href="httpps://datahub.com/adoption-stories" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Adoption Stories"></div><div class="title_c7DP">Adoption Stories</div></a></div><div><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Blog"></div><div class="title_c7DP">Blog</div></a></div><div><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Youtube"></div><div class="title_c7DP">Youtube</div></a></div></div></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/slack/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Join Slack"></div><div class="title_c7DP">Join Slack</div></a></div><div><a href="https://datahub.com/events" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Events"></div><div class="title_c7DP">Events</div></a></div><div><a href="https://datahub.com/champions/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Champions"></div><div class="title_c7DP">Champions</div></a></div><div><a href="https://datahub.com/share-your-journey/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Share Your Journey"></div><div class="title_c7DP">Share Your Journey</div></a></div></div></ul></div><a href="https://datahub.com/products/why-datahub-cloud/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
||
<style>
|
||
.cloud-cta {
|
||
color: var(--ifm-menu-color-active);
|
||
font-weight: 600;
|
||
background: linear-gradient(40deg, var(--ifm-menu-color-active), var(--ifm-menu-color-active));
|
||
background-size: 200% 100%;
|
||
-webkit-background-clip: text;
|
||
background-clip: text;
|
||
transition: background-image 0.3s ease;
|
||
}
|
||
.cloud-cta:hover {
|
||
color: transparent;
|
||
background: linear-gradient(40deg, var(--ifm-menu-color-active), #ff1493);
|
||
background-size: 200% 100%;
|
||
-webkit-background-clip: text;
|
||
background-clip: text;
|
||
animation: gradientShift 3s ease infinite;
|
||
}
|
||
@keyframes gradientShift {
|
||
0%, 100% { background-position: 0% 50%; }
|
||
50% { background-position: 100% 50%; }
|
||
}
|
||
</style>
|
||
<div class="cloud-cta">Get Cloud</div>
|
||
</a><a href="https://datahub.com/slack?utm_source=docs&utm_medium=header&utm_campaign=docs_header" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
||
<style>
|
||
.slack-logo:hover {
|
||
opacity: 0.8;
|
||
}
|
||
</style>
|
||
<img class="slack-logo" src="https://upload.wikimedia.org/wikipedia/commons/d/d5/Slack_icon_2019.svg" , alt="slack" , height="20px" style="margin: 10px 0 0 0;">
|
||
</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Getting Started</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/features">What Is DataHub?</a><button aria-label="Toggle the collapsible sidebar category 'What Is DataHub?'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/features">Features</a><button aria-label="Toggle the collapsible sidebar category 'Features'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>DataHub Cloud</div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/managed-datahub-overview">DataHub Cloud Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/welcome-acryl">Getting Started with DataHub Cloud</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authentication/guides/sso/initialize-oidc">Configure Single Sign-On</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/remote-executor/about">Remote Executor</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/datahub-api/entity-events-api">DataHub API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/slack/saas-slack-app">Slack</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge">Operator Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item saasOnly"><a class="menu__link" href="/docs/managed-datahub/approval-workflows">Change Proposals & Approval Workflows</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/chrome-extension">Cloud Chrome Extension</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item saasOnly"><a class="menu__link" href="/docs/managed-datahub/subscription-and-notification">Subscriptions & Notifications</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/release-notes/v_0_3_12">DataHub Cloud Release History</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Integrations</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-ingestion">Overview</a><button aria-label="Toggle the collapsible sidebar category 'Overview'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/cli-ingestion">Quickstart Guides</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-ingestion/source_overview">Sources</a><button aria-label="Toggle the collapsible sidebar category 'Sources'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/schedule_docs/intro">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>API & SDKs</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-modeling/metadata-model">DataHub's Open Metadata Standard</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/what-is-datahub/datahub-concepts">Concepts</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-standards">Metadata Standards</a><button aria-label="Toggle the collapsible sidebar category 'Metadata Standards'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/api/datahub-apis">APIs and SDKs Overview</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/api/graphql/overview">API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/as-a-library">Python SDK</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/metadata-integration/java/as-a-library">Java SDK</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/cli">DataHub CLI</a><button aria-label="Toggle the collapsible sidebar category 'DataHub CLI'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/act-on-metadata">DataHub Actions</a><button aria-label="Toggle the collapsible sidebar category 'DataHub Actions'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/api/tutorials/datasets">Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Admin</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authentication">Authentication</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authorization">Authorization</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/how/delete-metadata">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Deployment</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/deployment-guides">Deployment Guides</a><button aria-label="Toggle the collapsible sidebar category 'Deployment Guides'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/deploy/confluent-cloud">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Developers</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/architecture/architecture">Architecture</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/developers">Developing on DataHub</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/advanced/mcp-mcl">Advanced Guides</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/advanced/mcp-mcl">MetadataChangeProposal & MetadataChangeLog Events</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/advanced/writing-mcps">Saving MCPs to a File</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/metadata-modeling/extending-the-metadata-model">Extending the Metadata Model</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/advanced/api-tracing">API Tracing</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/datahub-web-react/src/app/analytics">React Analytics</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/docker/datahub-upgrade">Upgrade Docker Image</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/metadata-ingestion/adding-source">Adding a Metadata Ingestion Source</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/how/add-custom-ingestion-source">Using a Custom Ingestion Source</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/how/add-custom-data-platform">Adding a custom Dataset Data Platform</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/how/migrating-graph-service-implementation">Migrate Graph Service Implementation to Elasticsearch</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/advanced/field-path-spec-v2">SchemaFieldPath Specification (Version 2)</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/advanced/browse-paths-upgrade">Browse Paths Upgrade (August 2022)</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/browsev2/browse-paths-v2">Generating Browse Paths (V2)</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/metadata-ingestion/docs/dev_guides/reporting_telemetry">Datahub's Reporting Framework for Ingestion Job Telemetry</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/plugins">Plugins Guide</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/advanced/bootstrap-mcps">Bootstrap MetadataChangeProposals (MCPs)</a></li></ul></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Community</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/community">Community</a><button aria-label="Toggle the collapsible sidebar category 'Community'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/releases">Release History</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Advanced Guides</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Extending the Metadata Model</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Extending the Metadata Model</h1><p>You can extend the metadata model by either creating a new Entity or extending an existing one. Unsure if you need to
|
||
create a new entity or add an aspect to an existing entity? Read <a href="/docs/metadata-modeling/metadata-model">metadata-model</a> to understand
|
||
these two concepts prior to making changes.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="to-fork-or-not-to-fork">To fork or not to fork?<a href="#to-fork-or-not-to-fork" class="hash-link" aria-label="Direct link to To fork or not to fork?" title="Direct link to To fork or not to fork?"></a></h2><p>An important question that will arise once you've decided to extend the metadata model is whether you need to fork the main repo or not. Use the diagram below to understand how to make this decision.</p><p align="center"><img loading="lazy" width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/metadata-model-to-fork-or-not-to.png" class="img_ev3q"></p><p>The green lines represent pathways that will lead to lesser friction for you to maintain your code long term. The red lines represent higher risk of conflicts in the future. We are working hard to move the majority of model extension use-cases to no-code / low-code pathways to ensure that you can extend the core metadata model without having to maintain a custom fork of DataHub.</p><p>We will refer to the two options as the <strong>open-source fork</strong> and <strong>custom repository</strong> approaches in the rest of the document below.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="this-guide">This Guide<a href="#this-guide" class="hash-link" aria-label="Direct link to This Guide" title="Direct link to This Guide"></a></h2><p>This guide will outline what the experience of adding a new Entity should look like through a real example of adding the
|
||
Dashboard Entity. If you want to extend an existing Entity, you can skip directly to <a href="#step-3-define-custom-aspects-or-attach-existing-aspects-to-your-entity">Step 3</a>.</p><p>At a high level, an entity is made up of:</p><ol><li>A Key Aspect: Uniquely identifies an instance of an entity,</li><li>A list of specified Aspects, groups of related attributes that are attached to an entity.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="defining-an-entity">Defining an Entity<a href="#defining-an-entity" class="hash-link" aria-label="Direct link to Defining an Entity" title="Direct link to Defining an Entity"></a></h2><p>Now we'll walk through the steps required to create, ingest, and view your extensions to the metadata model. We will use
|
||
the existing "Dashboard" entity for purposes of illustration.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-define-the-entity-key-aspect"><a name="step_1"></a>Step 1: Define the Entity Key Aspect<a href="#step-1-define-the-entity-key-aspect" class="hash-link" aria-label="Direct link to step-1-define-the-entity-key-aspect" title="Direct link to step-1-define-the-entity-key-aspect"></a></h3><p>A key represents the fields that uniquely identify the entity. For those familiar with DataHub’s legacy architecture,
|
||
these fields were previously part of the Urn Java Class that was defined for each entity.</p><p>This struct will be used to generate a serialized string key, represented by an Urn. Each field in the key struct will
|
||
be converted into a single part of the Urn's tuple, in the order they are defined.</p><p>Let’s define a Key aspect for our new Dashboard entity.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">namespace com.linkedin.metadata.key</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">/**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Key for a Dashboard</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">@Aspect = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": "dashboardKey",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">record DashboardKey {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * The name of the dashboard tool such as looker, redash etc.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> @Searchable = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> dashboardTool: string</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Unique id for the dashboard. This id should be globally unique for a dashboarding tool even when there are multiple deployments of it. As an example, dashboard URL could be used here for Looker such as 'looker.linkedin.com/dashboards/1234'</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> dashboardId: string</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The Urn representation of the Key shown above would be:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">urn:li:dashboard:(<tool>,<id>)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Because they are aspects, keys need to be annotated with an @Aspect annotation, This instructs DataHub that this struct
|
||
can be a part of.</p><p>The key can also be annotated with the two index annotations: @Relationship and @Searchable. This instructs DataHub
|
||
infra to use the fields in the key to create relationships and index fields for search. See <a href="#step-3-define-custom-aspects-or-attach-existing-aspects-to-your-entity">Step 3</a> for more details on
|
||
the annotation model.</p><p><strong>Constraints</strong>: Note that each field in a Key Aspect MUST be of String or Enum type.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-create-the-new-entity-with-its-key-aspect"><a name="step_2"></a>Step 2: Create the new entity with its key aspect<a href="#step-2-create-the-new-entity-with-its-key-aspect" class="hash-link" aria-label="Direct link to step-2-create-the-new-entity-with-its-key-aspect" title="Direct link to step-2-create-the-new-entity-with-its-key-aspect"></a></h3><p>Define the entity within an <code>entity-registry.yml</code> file. Depending on your approach, the location of this file may vary. More on that in steps <a href="#step-4-choose-a-place-to-store-your-model-extension">4</a> and <a href="#step-5-attaching-your-non-key-aspects-to-the-entity">5</a>.</p><p>Example:</p><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> dashboard</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">doc</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> A container of related data assets.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">keyAspect</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> dashboardKey</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ul><li>name: The entity name/type, this will be present as a part of the Urn.</li><li>doc: A brief description of the entity.</li><li>keyAspect: The name of the Key Aspect defined in step 1. This name must match the value in the PDL annotation.</li></ul><h1></h1><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-define-custom-aspects-or-attach-existing-aspects-to-your-entity"><a name="step_3"></a>Step 3: Define custom aspects or attach existing aspects to your entity<a href="#step-3-define-custom-aspects-or-attach-existing-aspects-to-your-entity" class="hash-link" aria-label="Direct link to step-3-define-custom-aspects-or-attach-existing-aspects-to-your-entity" title="Direct link to step-3-define-custom-aspects-or-attach-existing-aspects-to-your-entity"></a></h3><p>Some aspects, like Ownership and GlobalTags, are reusable across entities. They can be included in an entity’s set of
|
||
aspects freely. To include attributes that are not included in an existing Aspect, a new Aspect must be created.</p><p>Let’s look at the DashboardInfo aspect as an example of what goes into a new aspect.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">namespace com.linkedin.dashboard</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">import com.linkedin.common.AccessLevel</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">import com.linkedin.common.ChangeAuditStamps</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">import com.linkedin.common.ChartUrn</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">import com.linkedin.common.Time</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">import com.linkedin.common.Url</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">import com.linkedin.common.CustomProperties</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">import com.linkedin.common.ExternalReference</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">/**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Information about a dashboard</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">@Aspect = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": "dashboardInfo"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">record DashboardInfo includes CustomProperties, ExternalReference {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Title of the dashboard</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> @Searchable = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "fieldType": "TEXT_WITH_PARTIAL_MATCHING",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "queryByDefault": true,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "enableAutocomplete": true,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "boostScore": 10.0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> title: string</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Detailed description about the dashboard</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> @Searchable = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "fieldType": "TEXT",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "queryByDefault": true,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "hasValuesFieldName": "hasDescription"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> description: string</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Charts in a dashboard</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> @Relationship = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "/*": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": "Contains",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "entityTypes": [ "chart" ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> charts: array[ChartUrn] = [ ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Captures information about who created/last modified/deleted this dashboard and when</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> lastModified: ChangeAuditStamps</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * URL for the dashboard. This could be used as an external link on DataHub to allow users access/view the dashboard</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> dashboardUrl: optional Url</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Access level for the dashboard</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> @Searchable = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "fieldType": "KEYWORD",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "addToFilters": true</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> access: optional AccessLevel</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * The time when this dashboard last refreshed</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> lastRefreshed: optional Time</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The Aspect has four key components: its properties, the @Aspect annotation, the @Searchable annotation and the
|
||
@Relationship annotation. Let’s break down each of these:</p><ul><li><strong>Aspect properties</strong>: The record’s properties can be declared as a field on the record, or by including another
|
||
record in the Aspect’s definition (<code>record DashboardInfo includes CustomProperties, ExternalReference {</code>). Properties
|
||
can be defined as PDL primitives, enums, records, or collections (
|
||
see <a href="https://linkedin.github.io/rest.li/pdl_schema" target="_blank" rel="noopener noreferrer">pdl schema documentation</a>)
|
||
references to other entities, of type Urn or optionally <code><Entity>Urn</code></li><li><strong>@Aspect annotation</strong>: Declares record is an Aspect and includes it when serializing an entity. Unlike the following
|
||
two annotations, @Aspect is applied to the entire record, rather than a specific field. Note, you can mark an aspect
|
||
as a timeseries aspect. Check out this <a href="/docs/metadata-modeling/metadata-model#timeseries-aspects">doc</a> for details.</li><li><strong>@Searchable annotation</strong>: This annotation can be applied to any primitive field or a map field to indicate that it
|
||
should be indexed in Elasticsearch and can be searched on. For a complete guide on using the search annotation, see
|
||
the annotation docs further down in this document.</li><li><strong>@Relationship annotation</strong>: These annotations create edges between the Entity’s Urn and the destination of the
|
||
annotated field when the entities are ingested. @Relationship annotations must be applied to fields of type Urn. In
|
||
the case of DashboardInfo, the <code>charts</code> field is an Array of Urns. The @Relationship annotation cannot be applied
|
||
directly to an array of Urns. That’s why you see the use of an Annotation override (<code>"/*":</code>) to apply the @Relationship
|
||
annotation to the Urn directly. Read more about overrides in the annotation docs further down on this page.</li><li><strong>@UrnValidation</strong>: This annotation can enforce constraints on Urn fields, including entity type restrictions and existence.</li></ul><p>After you create your Aspect, you need to attach to all the entities that it applies to.</p><p><strong>Constraints</strong>: Note that all aspects MUST be of type Record.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-choose-a-place-to-store-your-model-extension"><a name="step_4"></a>Step 4: Choose a place to store your model extension<a href="#step-4-choose-a-place-to-store-your-model-extension" class="hash-link" aria-label="Direct link to step-4-choose-a-place-to-store-your-model-extension" title="Direct link to step-4-choose-a-place-to-store-your-model-extension"></a></h3><p>At the beginning of this document, we walked you through a flow-chart that should help you decide whether you need to maintain a fork of the open source DataHub repo for your model extensions, or whether you can just use a model extension repository that can stay independent of the DataHub repo. Depending on what path you took, the place you store your aspect model files (the .pdl files) and the entity-registry files (the yaml file called <code>entity-registry.yaml</code> or <code>entity-registry.yml</code>) will vary.</p><ul><li>Open source Fork: Aspect files go under <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models" target="_blank" rel="noopener noreferrer"><code>metadata-models</code></a> module in the main repo, entity registry goes into <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/resources/entity-registry.yml" target="_blank" rel="noopener noreferrer"><code>metadata-models/src/main/resources/entity-registry.yml</code></a>. Read on for more details in <a href="#step-5-attaching-your-non-key-aspects-to-the-entity">Step 5</a>.</li><li>Custom repository: Read the <a href="/docs/metadata-models-custom">metadata-models-custom</a> documentation to learn how to store and version your aspect models and registry.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-attaching-your-non-key-aspects-to-the-entity"><a name="step_5"></a>Step 5: Attaching your non-key Aspect(s) to the Entity<a href="#step-5-attaching-your-non-key-aspects-to-the-entity" class="hash-link" aria-label="Direct link to step-5-attaching-your-non-key-aspects-to-the-entity" title="Direct link to step-5-attaching-your-non-key-aspects-to-the-entity"></a></h3><p>Attaching non-key aspects to an entity can be done simply by adding them to the entity registry yaml file. The location of this file differs based on whether you are following the oss-fork path or the custom-repository path.</p><p>Here is an minimal example of adding our new <code>DashboardInfo</code> aspect to the <code>Dashboard</code> entity.</p><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token key atrule">entities</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> dashboard</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">keyAspect</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> dashBoardKey</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">aspects</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># the name of the aspect must be the same as that on the @Aspect annotation on the class</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> dashboardInfo</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Previously, you were required to add all aspects for the entity into an Aspect union. You will see examples of this pattern throughout the code-base (e.g. <code>DatasetAspect</code>, <code>DashboardAspect</code> etc.). This is no longer required.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-oss-fork-approach-re-build-datahub-to-have-access-to-your-new-or-updated-entity"><a name="step_6"></a>Step 6 (Oss-Fork approach): Re-build DataHub to have access to your new or updated entity<a href="#step-6-oss-fork-approach-re-build-datahub-to-have-access-to-your-new-or-updated-entity" class="hash-link" aria-label="Direct link to step-6-oss-fork-approach-re-build-datahub-to-have-access-to-your-new-or-updated-entity" title="Direct link to step-6-oss-fork-approach-re-build-datahub-to-have-access-to-your-new-or-updated-entity"></a></h3><p>If you opted for the open-source fork approach, where you are editing models in the <code>metadata-models</code> repository of DataHub, you will need to re-build the DataHub metadata service using the steps below. If you are following the custom model repository approach, you just need to build your custom model repository and deploy it to a running metadata service instance to read and write metadata using your new model extensions.</p><p>Read on to understand how to re-build DataHub for the oss-fork option.</p><p><strong><em>NOTE</em></strong>: If you have updated any existing types or see an <code>Incompatible changes</code> warning when building, you will need to run
|
||
<code>./gradlew :metadata-service:restli-servlet-impl:build -Prest.model.compatibility=ignore</code>
|
||
before running <code>build</code>.</p><p>Then, run <code>./gradlew build</code> from the repository root to rebuild Datahub with access to your new entity.</p><p>Then, re-deploy metadata-service (gms), and mae-consumer and mce-consumer (optionally if you are running them unbundled). See <a href="/docs/docker">docker development</a> for details on how
|
||
to deploy during development. This will allow Datahub to read and write your new entity or extensions to existing entities, along with serving search and graph queries for that entity type.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="optional-step-7-use-custom-models-with-the-python-sdk"><a name="step_7"></a>(Optional) Step 7: Use custom models with the Python SDK<a href="#optional-step-7-use-custom-models-with-the-python-sdk" class="hash-link" aria-label="Direct link to optional-step-7-use-custom-models-with-the-python-sdk" title="Direct link to optional-step-7-use-custom-models-with-the-python-sdk"></a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Local CLI</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Custom Models Package</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><p>If you're purely using the custom models locally, you can use a local development-mode install of the DataHub CLI.</p><p>Install the DataHub CLI locally by following the <a href="/docs/metadata-ingestion/developing">developer instructions</a>.
|
||
The <code>./gradlew build</code> command already generated the avro schemas for your local ingestion cli tool to use.
|
||
After following the developing guide, you should be able to emit your new event using the local DataHub CLI.</p></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><p>If you want to use your custom models beyond your local machine without forking DataHub, then you can generate a custom model package that can be installed from other places.</p><p>This package should be installed alongside the base <code>acryl-datahub</code> package, and its metadata models will take precedence over the default ones.</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">$ </span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">cd</span><span class="token plain"> metadata-ingestion</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">$ </span><span class="token punctuation" style="color:rgb(199, 146, 234)">..</span><span class="token plain">/gradlew customPackageGenerate -Ppackage_name</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">my-company-datahub-models -Ppackage_version</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token string" style="color:rgb(195, 232, 141)">"0.0.1"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token plain">bunch of log lines</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Successfully built my-company-datahub-models-0.0.1.tar.gz and acryl_datahub_cloud-0.0.1-py3-none-any.whl</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Generated package at custom-package/my-company-datahub-models</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">This package should be installed alongside the main acryl-datahub package.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Install the custom package locally with </span><span class="token variable" style="color:rgb(191, 199, 213)">`</span><span class="token variable" style="color:rgb(191, 199, 213)">pip </span><span class="token variable function" style="color:rgb(130, 170, 255)">install</span><span class="token variable" style="color:rgb(191, 199, 213)"> custom-package/my-company-datahub-models</span><span class="token variable" style="color:rgb(191, 199, 213)">`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">To </span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">enable</span><span class="token plain"> others to use it, share the </span><span class="token function" style="color:rgb(130, 170, 255)">file</span><span class="token plain"> at custom-package/my-company-datahub-models/dist/</span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token plain">wheel file</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain">.whl and have them </span><span class="token function" style="color:rgb(130, 170, 255)">install</span><span class="token plain"> it with </span><span class="token variable" style="color:rgb(191, 199, 213)">`</span><span class="token variable" style="color:rgb(191, 199, 213)">pip </span><span class="token variable function" style="color:rgb(130, 170, 255)">install</span><span class="token variable" style="color:rgb(191, 199, 213)"> </span><span class="token variable operator" style="color:rgb(137, 221, 255)"><</span><span class="token variable" style="color:rgb(191, 199, 213)">wheel file</span><span class="token variable operator" style="color:rgb(137, 221, 255)">></span><span class="token variable" style="color:rgb(191, 199, 213)">.whl</span><span class="token variable" style="color:rgb(191, 199, 213)">`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Alternatively, publish it to PyPI with </span><span class="token variable" style="color:rgb(191, 199, 213)">`</span><span class="token variable" style="color:rgb(191, 199, 213)">twine upload custom-package/my-company-datahub-models/dist/*</span><span class="token variable" style="color:rgb(191, 199, 213)">`</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This will generate some Python build artifacts, which you can distribute within your team or publish to PyPI.
|
||
The command output contains additional details and exact CLI commands you can use.</p><p>Once this package is installed, you can use the DataHub CLI as normal, and it will use your custom models.
|
||
You'll also be able to import those models, with IDE support, by changing your imports.</p><div class="language-diff codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-diff codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token deleted-sign deleted prefix deleted" style="color:rgb(255, 85, 114)">-</span><span class="token deleted-sign deleted line" style="color:rgb(255, 85, 114)"> from datahub.metadata.schema_classes import DatasetPropertiesClass</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token deleted-sign deleted line" style="color:rgb(255, 85, 114)"></span><span class="token inserted-sign inserted prefix inserted" style="color:rgb(195, 232, 141)">+</span><span class="token inserted-sign inserted line" style="color:rgb(195, 232, 141)"> from my_company_datahub_models.metadata.schema_classes import DatasetPropertiesClass</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="optional-step-8-extend-the-datahub-frontend-to-view-your-entity-in-graphql--react"><a name="step_8"></a>(Optional) Step 8: Extend the DataHub frontend to view your entity in GraphQL & React<a href="#optional-step-8-extend-the-datahub-frontend-to-view-your-entity-in-graphql--react" class="hash-link" aria-label="Direct link to optional-step-8-extend-the-datahub-frontend-to-view-your-entity-in-graphql--react" title="Direct link to optional-step-8-extend-the-datahub-frontend-to-view-your-entity-in-graphql--react"></a></h3><p>If you are extending an entity with additional aspects, and you can use the auto-render specifications to automatically render these aspects to your satisfaction, you do not need to write any custom code.</p><p>However, if you want to write specific code to render your model extensions, or if you introduced a whole new entity and want to give it its own page, you will need to write custom React and Grapqhl code to view and mutate your entity in GraphQL or React. For
|
||
instructions on how to start extending the GraphQL graph, see <a href="/docs/datahub-graphql-core">graphql docs</a>. Once you’ve done that, you can follow the guide <a href="/docs/datahub-web-react">here</a> to add your entity into the React UI.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="metadata-annotations">Metadata Annotations<a href="#metadata-annotations" class="hash-link" aria-label="Direct link to Metadata Annotations" title="Direct link to Metadata Annotations"></a></h2><p>There are four core annotations that DataHub recognizes:</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="entity">@Entity<a href="#entity" class="hash-link" aria-label="Direct link to @Entity" title="Direct link to @Entity"></a></h4><p><strong>Legacy</strong>
|
||
This annotation is applied to each Entity Snapshot record, such as DashboardSnapshot.pdl. Each one that is included in
|
||
the root Snapshot.pdl model must have this annotation.</p><p>It takes the following parameters:</p><ul><li><strong>name</strong>: string - A common name used to identify the entity. Must be unique among all entities DataHub is aware of.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="example">Example<a href="#example" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example"></a></h5><div class="language-aidl codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-aidl codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">@Entity = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> // name used when referring to the entity in APIs.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> String name;</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="aspect">@Aspect<a href="#aspect" class="hash-link" aria-label="Direct link to @Aspect" title="Direct link to @Aspect"></a></h4><p>This annotation is applied to each Aspect record, such as DashboardInfo.pdl. Each aspect that is included in an entity’s
|
||
set of aspects in the <code>entity-registry.yml</code> must have this annotation.</p><p>It takes the following parameters:</p><ul><li><strong>name</strong>: string - A common name used to identify the Aspect. Must be unique among all aspects DataHub is aware of.</li><li><strong>type</strong>: string (optional) - set to "timeseries" to mark this aspect as timeseries. Check out
|
||
this <a href="/docs/metadata-modeling/metadata-model#timeseries-aspects">doc</a> for details.</li><li><strong>autoRender</strong>: boolean (optional) - defaults to false. When set to true, the aspect will automatically be displayed
|
||
on entity pages in a tab using a default renderer. <strong><em>This is currently only supported for Charts, Dashboards, DataFlows, DataJobs, Datasets, Domains, and GlossaryTerms</em></strong>.</li><li><strong>renderSpec</strong>: RenderSpec (optional) - config for autoRender aspects that controls how they are displayed. <strong><em>This is currently only supported for Charts, Dashboards, DataFlows, DataJobs, Datasets, Domains, and GlossaryTerms</em></strong>. Contains three fields:<ul><li><strong>displayType</strong>: One of <code>tabular</code>, <code>properties</code>. Tabular should be used for a list of data elements, properties for a single data bag.</li><li><strong>displayName</strong>: How the aspect should be referred to in the UI. Determines the name of the tab on the entity page.</li><li><strong>key</strong>: For <code>tabular</code> aspects only. Specifies the key in which the array to render may be found.</li></ul></li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="example-1">Example<a href="#example-1" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example"></a></h5><div class="language-aidl codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-aidl codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">@Aspect = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> // name used when referring to the aspect in APIs.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> String name;</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="searchable">@Searchable<a href="#searchable" class="hash-link" aria-label="Direct link to @Searchable" title="Direct link to @Searchable"></a></h4><p>This annotation is applied to fields inside an Aspect. It instructs DataHub to index the field so it can be retrieved
|
||
via the search APIs.</p><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span><mdxadmonitiontitle>If you are adding @Searchable to a field that already has data, you'll want to restore indices <a href="/docs/api/restli/restore-indices/">via api</a> or <a href="https://github.com/datahub-project/datahub/blob/master/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java" target="_blank" rel="noopener noreferrer">via upgrade step</a> to have it be populated with existing data.</mdxadmonitiontitle></div><div class="admonitionContent_S0QG"><p>It takes the following parameters:</p><ul><li><p><strong>fieldType</strong>: string - The settings for how each field is indexed is defined by the field type. Each field type is
|
||
associated with a set of analyzers Elasticsearch will use to tokenize the field. Such sets are defined in the
|
||
MappingsBuider, which generates the mappings for the index for each entity given the fields with the search
|
||
annotations. To customize the set of analyzers used to index a certain field, you must add a new field type and define
|
||
the set of mappings to be applied in the MappingsBuilder.</p><p>Thus far, we have implemented 11 fieldTypes:</p><ol><li><p><em>KEYWORD</em> - Short text fields that only support exact matches, often used only for filtering</p></li><li><p><em>TEXT</em> - Text fields delimited by spaces/slashes/periods. Default field type for string variables.</p></li><li><p><em>TEXT_PARTIAL</em> - Text fields delimited by spaces/slashes/periods with partial matching support. Note, partial
|
||
matching is expensive, so this field type should not be applied to fields with long values (like description)</p></li><li><p><em>WORD_GRAM</em> - Text fields delimited by spaces, slashes, periods, dashes, or underscores with partial matching AND
|
||
word gram support. That is, the text will be split by the delimiters and can be matched with delimited queries
|
||
matching two, three, or four length tokens in addition to single tokens. As with partial match, this type is
|
||
expensive, so should not be applied to fields with long values such as description.</p></li><li><p><em>BROWSE_PATH</em> - Field type for browse paths. Applies specific mappings for slash delimited paths.</p></li><li><p><em>URN</em> - Urn fields where each sub-component inside the urn is indexed. For instance, for a data platform urn like
|
||
"urn:li:dataplatform:kafka", it will index the platform name "kafka" and ignore the common components</p></li><li><p><em>URN_PARTIAL</em> - Urn fields where each sub-component inside the urn is indexed with partial matching support.</p></li><li><p><em>BOOLEAN</em> - Boolean fields used for filtering.</p></li><li><p><em>COUNT</em> - Count fields used for filtering.</p></li><li><p><em>DATETIME</em> - Datetime fields used to represent timestamps.</p></li><li><p><em>OBJECT</em> - Each property in an object will become an extra column in Elasticsearch and can be referenced as
|
||
<code>field.property</code> in queries. You should be careful to not use it on objects with many properties as it can cause a
|
||
mapping explosion in Elasticsearch.</p></li></ol></li><li><p><strong>fieldName</strong>: string (optional) - The name of the field in search index document. Defaults to the field name where
|
||
the annotation resides.</p></li><li><p><strong>queryByDefault</strong>: boolean (optional) - Whether we should match the field for the default search query. True by
|
||
default for text and urn fields.</p></li><li><p><strong>enableAutocomplete</strong>: boolean (optional) - Whether we should use the field for autocomplete. Defaults to false</p></li><li><p><strong>addToFilters</strong>: boolean (optional) - Whether or not to add field to filters. Defaults to false</p></li><li><p><strong>boostScore</strong>: double (optional) - Boost multiplier to the match score. Matches on fields with higher boost score
|
||
ranks higher.</p></li><li><p><strong>hasValuesFieldName</strong>: string (optional) - If set, add an index field of the given name that checks whether the field
|
||
exists</p></li><li><p><strong>numValuesFieldName</strong>: string (optional) - If set, add an index field of the given name that checks the number of
|
||
elements</p></li><li><p><strong>weightsPerFieldValue</strong>: map<!-- -->[object, double]<!-- --> (optional) - Weights to apply to score for a given value.</p></li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="example-2">Example<a href="#example-2" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example"></a></h5><p>Let’s take a look at a real world example using the <code>title</code> field of <code>DashboardInfo.pdl</code>:</p><div class="language-aidl codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-aidl codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">record DashboardInfo {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Title of the dashboard</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> @Searchable = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "fieldType": "TEXT_PARTIAL",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "enableAutocomplete": true,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "boostScore": 10.0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> title: string</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ....</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This annotation is saying that we want to index the title field in Elasticsearch. We want to support partial matches on
|
||
the title, so queries for <code>Cust</code> should return a Dashboard with the title <code>Customers</code>. <code>enableAutocomplete</code> is set to
|
||
true, meaning that we can autocomplete on this field when typing into the search bar. Finally, a boostScore of 10 is
|
||
provided, meaning that we should prioritize matches to title over matches to other fields, such as description, when
|
||
ranking.</p><p>Now, when Datahub ingests Dashboards, it will index the Dashboard’s title in Elasticsearch. When a user searches for
|
||
Dashboards, that query will be used to search on the title index and matching Dashboards will be returned.</p><p>Note, when @Searchable annotation is applied to a map, it will convert it into a list with "key.toString()
|
||
=value.toString()" as elements. This allows us to index map fields, while not increasing the number of columns indexed.
|
||
This way, the keys can be queried by <code>aMapField:key1=value1</code>.</p><p>You can change this behavior by specifying the fieldType as OBJECT in the @Searchable annotation. It will put each key
|
||
into a column in Elasticsearch instead of an array of serialized kay-value pairs. This way the query would look more
|
||
like <code>aMapField.key1:value1</code>. As this method will increase the number of columns with each unique key - large maps can
|
||
cause a mapping explosion in Elasticsearch. You should <em>not</em> use the object fieldType if you expect your maps to get
|
||
large.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="relationship">@Relationship<a href="#relationship" class="hash-link" aria-label="Direct link to @Relationship" title="Direct link to @Relationship"></a></h4><p>This annotation is applied to fields inside an Aspect. This annotation creates edges between an Entity’s Urn and the
|
||
destination of the annotated field when the Entity is ingested. @Relationship annotations must be applied to fields of
|
||
type Urn.</p><p>It takes the following parameters:</p><ul><li><strong>name</strong>: string - A name used to identify the Relationship type.</li><li><strong>entityTypes</strong>: array<!-- -->[string]<!-- --> (Optional) - A list of entity types that are valid values for the foreign-key
|
||
relationship field.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="example-3">Example<a href="#example-3" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example"></a></h5><p>Let’s take a look at a real world example to see how this annotation is used. The <code>Owner.pdl</code> struct is referenced by
|
||
the <code>Ownership.pdl</code> aspect. <code>Owned.pdl</code> contains a relationship to a CorpUser or CorpGroup:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">namespace com.linkedin.common</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">/**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Ownership information</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">record Owner {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> @Relationship = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": "OwnedBy",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "entityTypes": [ "corpUser", "corpGroup" ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> owner: Urn</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This annotation says that when we ingest an Entity with an Ownership Aspect, DataHub will create an OwnedBy relationship
|
||
between that entity and the CorpUser or CorpGroup who owns it. This will be queryable using the Relationships resource
|
||
in both the forward and inverse directions.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="urnvalidation">@UrnValidation<a href="#urnvalidation" class="hash-link" aria-label="Direct link to @UrnValidation" title="Direct link to @UrnValidation"></a></h4><p>This annotation can be applied to Urn fields inside an aspect. The annotation can optionally perform one or more of the following:</p><ul><li>Enforce that the URN exists</li><li>Enforce stricter URN validation</li><li>Restrict the URN to specific entity types</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="example-4">Example<a href="#example-4" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example"></a></h5><p>Using this example from StructuredPropertyDefinition, we are enforcing that the valueType URN must exist,
|
||
it must follow stricter Urn encoding logic, and it can only be of entity type <code>dataType</code>.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain"> @UrnValidation = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "exist": true,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "strict": true,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "entityTypes": [ "dataType" ],</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> valueType: Urn</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="annotating-collections--annotation-overrides">Annotating Collections & Annotation Overrides<a href="#annotating-collections--annotation-overrides" class="hash-link" aria-label="Direct link to Annotating Collections & Annotation Overrides" title="Direct link to Annotating Collections & Annotation Overrides"></a></h4><p>You will not always be able to apply annotations to a primitive field directly. This may be because the field is wrapped
|
||
in an Array, or because the field is part of a shared struct that many entities reference. In these cases, you need to
|
||
use annotation overrides. An override is done by specifying a fieldPath to the target field inside the annotation, like
|
||
so:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Charts in a dashboard</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> @Relationship = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "/*": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": "Contains",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "entityTypes": [ "chart" ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> charts: array[ChartUrn] = [ ]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This override applies the relationship annotation to each element in the Array, rather than the array itself. This
|
||
allows a unique Relationship to be created for between the Dashboard and each of its charts.</p><p>Another example can be seen in the case of tags. In this case, TagAssociation.pdl has a @Searchable annotation:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain"> @Searchable = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "fieldName": "tags",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "fieldType": "URN_WITH_PARTIAL_MATCHING",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "queryByDefault": true,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "hasValuesFieldName": "hasTags"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> tag: TagUrn</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>At the same time, SchemaField overrides that annotation to allow for searching for tags applied to schema fields
|
||
specifically. To do this, it overrides the Searchable annotation applied to the <code>tag</code> field of <code>TagAssociation</code> and
|
||
replaces it with its own- this has a different boostScore and a different fieldName.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Tags associated with the field</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> @Searchable = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "/tags/*/tag": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "fieldName": "fieldTags",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "fieldType": "URN_WITH_PARTIAL_MATCHING",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "queryByDefault": true,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "boostScore": 0.5</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> globalTags: optional GlobalTags</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>As a result, you can issue a query specifically for tags on Schema Fields via <code>fieldTags:<tag_name></code> or tags directly
|
||
applied to an entity via <code>tags:<tag_name></code>. Since both have <code>queryByDefault</code> set to true, you can also search for
|
||
entities with either of these properties just by searching for the tag name.</p></div></div></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="slackUtm_uoBr"><div class="slackUtm_uoBr"><hr>Need more help? Join the conversation in <a href="https://datahub.com/slack?utm_source=docs&utm_medium=footer&utm_campaign=docs_footer&utm_content=docs/modeling/extending-the-metadata-model">Slack!</a></div></div><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/datahub-project/datahub/blob/master/docs/modeling/extending-the-metadata-model.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_VsjB"></div></div></footer><div class="feedbackWrapper_mUHF"><div class="feedbackWidget_PX4d"><div class="feedbackButtons_wn3V"><strong>Is this page helpful?</strong><div><button class="feedbackButton_UgQs"><span role="img" aria-label="like" class="anticon anticon-like"><svg viewBox="64 64 896 896" focusable="false" data-icon="like" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M885.9 533.7c16.8-22.2 26.1-49.4 26.1-77.7 0-44.9-25.1-87.4-65.5-111.1a67.67 67.67 0 00-34.3-9.3H572.4l6-122.9c1.4-29.7-9.1-57.9-29.5-79.4A106.62 106.62 0 00471 99.9c-52 0-98 35-111.8 85.1l-85.9 311H144c-17.7 0-32 14.3-32 32v364c0 17.7 14.3 32 32 32h601.3c9.2 0 18.2-1.8 26.5-5.4 47.6-20.3 78.3-66.8 78.3-118.4 0-12.6-1.8-25-5.4-37 16.8-22.2 26.1-49.4 26.1-77.7 0-12.6-1.8-25-5.4-37 16.8-22.2 26.1-49.4 26.1-77.7-.2-12.6-2-25.1-5.6-37.1zM184 852V568h81v284h-81zm636.4-353l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 16.5-7.2 32.2-19.6 43l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 16.5-7.2 32.2-19.6 43l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 22.4-13.2 42.6-33.6 51.8H329V564.8l99.5-360.5a44.1 44.1 0 0142.2-32.3c7.6 0 15.1 2.2 21.1 6.7 9.9 7.4 15.2 18.6 14.6 30.5l-9.6 198.4h314.4C829 418.5 840 436.9 840 456c0 16.5-7.2 32.1-19.6 43z"></path></svg></span></button><button class="feedbackButton_UgQs"><span role="img" aria-label="dislike" class="anticon anticon-dislike"><svg viewBox="64 64 896 896" focusable="false" data-icon="dislike" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M885.9 490.3c3.6-12 5.4-24.4 5.4-37 0-28.3-9.3-55.5-26.1-77.7 3.6-12 5.4-24.4 5.4-37 0-28.3-9.3-55.5-26.1-77.7 3.6-12 5.4-24.4 5.4-37 0-51.6-30.7-98.1-78.3-118.4a66.1 66.1 0 00-26.5-5.4H144c-17.7 0-32 14.3-32 32v364c0 17.7 14.3 32 32 32h129.3l85.8 310.8C372.9 889 418.9 924 470.9 924c29.7 0 57.4-11.8 77.9-33.4 20.5-21.5 31-49.7 29.5-79.4l-6-122.9h239.9c12.1 0 23.9-3.2 34.3-9.3 40.4-23.5 65.5-66.1 65.5-111 0-28.3-9.3-55.5-26.1-77.7zM184 456V172h81v284h-81zm627.2 160.4H496.8l9.6 198.4c.6 11.9-4.7 23.1-14.6 30.5-6.1 4.5-13.6 6.8-21.1 6.7a44.28 44.28 0 01-42.2-32.3L329 459.2V172h415.4a56.85 56.85 0 0133.6 51.8c0 9.7-2.3 18.9-6.9 27.3l-13.9 25.4 21.9 19a56.76 56.76 0 0119.6 43c0 9.7-2.3 18.9-6.9 27.3l-13.9 25.4 21.9 19a56.76 56.76 0 0119.6 43c0 9.7-2.3 18.9-6.9 27.3l-14 25.5 21.9 19a56.76 56.76 0 0119.6 43c0 19.1-11 37.5-28.8 48.4z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/advanced/writing-mcps"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Saving MCPs to a File</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/advanced/api-tracing"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">API Tracing</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-fork-or-not-to-fork" class="table-of-contents__link toc-highlight">To fork or not to fork?</a></li><li><a href="#this-guide" class="table-of-contents__link toc-highlight">This Guide</a></li><li><a href="#defining-an-entity" class="table-of-contents__link toc-highlight">Defining an Entity</a><ul><li><a href="#step-1-define-the-entity-key-aspect" class="table-of-contents__link toc-highlight"><a name="step_1"></a>Step 1: Define the Entity Key Aspect</a></li><li><a href="#step-2-create-the-new-entity-with-its-key-aspect" class="table-of-contents__link toc-highlight"><a name="step_2"></a>Step 2: Create the new entity with its key aspect</a></li><li><a href="#step-3-define-custom-aspects-or-attach-existing-aspects-to-your-entity" class="table-of-contents__link toc-highlight"><a name="step_3"></a>Step 3: Define custom aspects or attach existing aspects to your entity</a></li><li><a href="#step-4-choose-a-place-to-store-your-model-extension" class="table-of-contents__link toc-highlight"><a name="step_4"></a>Step 4: Choose a place to store your model extension</a></li><li><a href="#step-5-attaching-your-non-key-aspects-to-the-entity" class="table-of-contents__link toc-highlight"><a name="step_5"></a>Step 5: Attaching your non-key Aspect(s) to the Entity</a></li><li><a href="#step-6-oss-fork-approach-re-build-datahub-to-have-access-to-your-new-or-updated-entity" class="table-of-contents__link toc-highlight"><a name="step_6"></a>Step 6 (Oss-Fork approach): Re-build DataHub to have access to your new or updated entity</a></li><li><a href="#optional-step-7-use-custom-models-with-the-python-sdk" class="table-of-contents__link toc-highlight"><a name="step_7"></a>(Optional) Step 7: Use custom models with the Python SDK</a></li><li><a href="#optional-step-8-extend-the-datahub-frontend-to-view-your-entity-in-graphql--react" class="table-of-contents__link toc-highlight"><a name="step_8"></a>(Optional) Step 8: Extend the DataHub frontend to view your entity in GraphQL & React</a></li></ul></li><li><a href="#metadata-annotations" class="table-of-contents__link toc-highlight">Metadata Annotations</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/docs/">Introduction</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/quickstart">Quickstart</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://datahub.com/slack" target="_blank" rel="noopener noreferrer" class="footer__link-item">Slack<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="footer__link-item">YouTube<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="footer__link-item">Blog<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/docs/townhalls">Town Halls</a></li><li class="footer__item"><a href="https://datahub.com/adoption-stories/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Adoption<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://demo.datahub.com/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Demo</a></li><li class="footer__item"><a href="https://feature-requests.datahubproject.io/roadmap" target="_blank" rel="noopener noreferrer" class="footer__link-item">Roadmap<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/docs/contributing">Contributing</a></li><li class="footer__item"><a href="https://github.com/datahub-project/datahub" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://feature-requests.datahubproject.io/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Feature Requests<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2015-2025 DataHub Project Authors.</div></div></div></footer></div>
|
||
<script src="/assets/js/runtime~main.50e13f51.js"></script>
|
||
<script src="/assets/js/main.edc0853c.js"></script>
|
||
</body>
|
||
</html> |