mirror of
https://github.com/datahub-project/datahub.git
synced 2025-07-07 01:00:41 +00:00
175 lines
143 KiB
HTML
175 lines
143 KiB
HTML
<!doctype html>
|
||
<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-docs/modeling/metadata-model" data-has-hydrated="false">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="generator" content="Docusaurus v2.4.3">
|
||
<title data-rh="true">The Metadata Model | DataHub</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://docs.datahub.com/docs/metadata-modeling/metadata-model"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="The Metadata Model | DataHub"><meta data-rh="true" name="description" content="DataHub takes a schema-first approach to modeling metadata. We use the open-source Pegasus schema language (PDL) extended with a custom set of annotations to model metadata. The DataHub storage, serving, indexing and ingestion layer operates directly on top of the metadata model and supports strong types all the way from the client to the storage layer."><meta data-rh="true" property="og:description" content="DataHub takes a schema-first approach to modeling metadata. We use the open-source Pegasus schema language (PDL) extended with a custom set of annotations to model metadata. The DataHub storage, serving, indexing and ingestion layer operates directly on top of the metadata model and supports strong types all the way from the client to the storage layer."><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://docs.datahub.com/docs/metadata-modeling/metadata-model"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/metadata-modeling/metadata-model" hreflang="en"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/metadata-modeling/metadata-model" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://RK0UG797F3-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/learn/rss.xml" title="DataHub RSS Feed">
|
||
<link rel="alternate" type="application/atom+xml" href="/learn/atom.xml" title="DataHub Atom Feed">
|
||
|
||
<link rel="preconnect" href="https://www.google-analytics.com">
|
||
<link rel="preconnect" href="https://www.googletagmanager.com">
|
||
<script async src="https://www.googletagmanager.com/gtag/js?id=G-PKGVLETT4C"></script>
|
||
<script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-PKGVLETT4C",{})</script>
|
||
<link rel="preconnect" href="https://www.googletagmanager.com">
|
||
<script>window.dataLayer=window.dataLayer||[]</script>
|
||
<script>!function(e,t,a,n,g){e[n]=e[n]||[],e[n].push({"gtm.start":(new Date).getTime(),event:"gtm.js"});var m=t.getElementsByTagName(a)[0],r=t.createElement(a);r.async=!0,r.src="https://www.googletagmanager.com/gtm.js?id=GTM-5M8T9HNN",m.parentNode.insertBefore(r,m)}(window,document,"script","dataLayer")</script>
|
||
|
||
|
||
<link rel="search" type="application/opensearchdescription+xml" title="DataHub" href="/opensearch.xml">
|
||
|
||
|
||
|
||
|
||
<meta httpequiv="Content-Security-Policy" content="frame-ancestors 'self' https://*.acryl.io https://acryldata.io http://localhost:*">
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap">
|
||
<script src="https://tools.luckyorange.com/core/lo.js?site-id=28ea8a38" async defer="defer"></script>
|
||
<script src="/scripts/rb2b.js" async defer="defer"></script>
|
||
<script src="https://app.revenuehero.io/scheduler.min.js"></script>
|
||
<script src="https://tag.clearbitscripts.com/v1/pk_2e321cabe30432a5c44c0424781aa35f/tags.js" referrerpolicy="strict-origin-when-cross-origin"></script>
|
||
<script src="/scripts/reo.js"></script>
|
||
<script id="runllm-widget-script" type="module" src="https://widget.runllm.com" crossorigin="true" runllm-name="DataHub" runllm-assistant-id="81" runllm-position="BOTTOM_RIGHT" runllm-keyboard-shortcut="Mod+j" runllm-preset="docusaurus" runllm-theme-color="#1890FF" runllm-brand-logo="https://docs.datahub.com/img/datahub-logo-color-mark.svg" runllm-community-url="https://datahub.com/slack" runllm-community-type="slack" runllm-disable-ask-a-person="true" async></script><link rel="stylesheet" href="/assets/css/styles.d8fe2eb8.css">
|
||
<link rel="preload" href="/assets/js/runtime~main.50e13f51.js" as="script">
|
||
<link rel="preload" href="/assets/js/main.edc0853c.js" as="script">
|
||
</head>
|
||
<body class="navigation-with-keyboard">
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-5M8T9HNN" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
|
||
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
|
||
<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:transparent;color:#ffffff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"><div class="shimmer-banner"><p>DataHub Secures $35 Million Series B</p><a href="https://datahub.com/news/series-b-announcement/" target="_blank" class="button"><div>Read the announcement<span> →</span></div></a></div></div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a href="https://datahub.com" target="_blank" rel="noopener noreferrer" class="navbar__brand"><div class="navbar__logo"><img src="/img/datahub-logo-color-light-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/datahub-logo-color-dark-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link versionNavItem_cbn8">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/metadata-modeling/metadata-model">Next</a></li><li><a class="dropdown__link" href="/docs/1.1.0/metadata-modeling/metadata-model">1.1.0</a></li><li><hr class="dropdown-separator" style="margin: 0.4rem;"></li><li><div class="dropdown__link"><b>Archived versions</b></div></li><li>
|
||
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/features">1.0.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/0.15.0/features">0.15.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-8jkm4uler-acryldata.vercel.app/docs/0.14.1/features">0.14.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-eue2qafvn-acryldata.vercel.app/docs/features">0.14.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-psat3nzgi-acryldata.vercel.app/docs/features">0.13.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-lzxh86531-acryldata.vercel.app/docs/features">0.13.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-2uuxmgza2-acryldata.vercel.app/docs/features">0.12.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-irpoe2osc-acryldata.vercel.app/docs/features">0.11.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-1gv2yzn9d-acryldata.vercel.app/docs/features">0.10.5
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li></ul></div></div><div class="navbar__items navbar__items--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/docs">Docs</a><a class="navbar__item navbar__link" href="/integrations">Integrations</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/weekly-demo" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Weekly Demo"></div><div class="title_c7DP">Weekly Demo</div></a></div><div><a href="https://datahub.com/use-cases" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-forum.png" alt="Use Cases"></div><div class="title_c7DP">Use Cases</div></a></div><div><a href="httpps://datahub.com/adoption-stories" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Adoption Stories"></div><div class="title_c7DP">Adoption Stories</div></a></div><div><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Blog"></div><div class="title_c7DP">Blog</div></a></div><div><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Youtube"></div><div class="title_c7DP">Youtube</div></a></div></div></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/slack/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Join Slack"></div><div class="title_c7DP">Join Slack</div></a></div><div><a href="https://datahub.com/events" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Events"></div><div class="title_c7DP">Events</div></a></div><div><a href="https://datahub.com/champions/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Champions"></div><div class="title_c7DP">Champions</div></a></div><div><a href="https://datahub.com/share-your-journey/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Share Your Journey"></div><div class="title_c7DP">Share Your Journey</div></a></div></div></ul></div><a href="https://datahub.com/products/why-datahub-cloud/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
||
<style>
|
||
.cloud-cta {
|
||
color: var(--ifm-menu-color-active);
|
||
font-weight: 600;
|
||
background: linear-gradient(40deg, var(--ifm-menu-color-active), var(--ifm-menu-color-active));
|
||
background-size: 200% 100%;
|
||
-webkit-background-clip: text;
|
||
background-clip: text;
|
||
transition: background-image 0.3s ease;
|
||
}
|
||
.cloud-cta:hover {
|
||
color: transparent;
|
||
background: linear-gradient(40deg, var(--ifm-menu-color-active), #ff1493);
|
||
background-size: 200% 100%;
|
||
-webkit-background-clip: text;
|
||
background-clip: text;
|
||
animation: gradientShift 3s ease infinite;
|
||
}
|
||
@keyframes gradientShift {
|
||
0%, 100% { background-position: 0% 50%; }
|
||
50% { background-position: 100% 50%; }
|
||
}
|
||
</style>
|
||
<div class="cloud-cta">Get Cloud</div>
|
||
</a><a href="https://datahub.com/slack?utm_source=docs&utm_medium=header&utm_campaign=docs_header" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
||
<style>
|
||
.slack-logo:hover {
|
||
opacity: 0.8;
|
||
}
|
||
</style>
|
||
<img class="slack-logo" src="https://upload.wikimedia.org/wikipedia/commons/d/d5/Slack_icon_2019.svg" , alt="slack" , height="20px" style="margin: 10px 0 0 0;">
|
||
</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Getting Started</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/features">What Is DataHub?</a><button aria-label="Toggle the collapsible sidebar category 'What Is DataHub?'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/features">Features</a><button aria-label="Toggle the collapsible sidebar category 'Features'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>DataHub Cloud</div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/managed-datahub-overview">DataHub Cloud Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/welcome-acryl">Getting Started with DataHub Cloud</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authentication/guides/sso/initialize-oidc">Configure Single Sign-On</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/remote-executor/about">Remote Executor</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/datahub-api/entity-events-api">DataHub API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/slack/saas-slack-app">Slack</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge">Operator Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item saasOnly"><a class="menu__link" href="/docs/managed-datahub/approval-workflows">Change Proposals & Approval Workflows</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/chrome-extension">Cloud Chrome Extension</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item saasOnly"><a class="menu__link" href="/docs/managed-datahub/subscription-and-notification">Subscriptions & Notifications</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/release-notes/v_0_3_12">DataHub Cloud Release History</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Integrations</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-ingestion">Overview</a><button aria-label="Toggle the collapsible sidebar category 'Overview'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/cli-ingestion">Quickstart Guides</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-ingestion/source_overview">Sources</a><button aria-label="Toggle the collapsible sidebar category 'Sources'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/schedule_docs/intro">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>API & SDKs</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/metadata-modeling/metadata-model">DataHub's Open Metadata Standard</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/metadata-modeling/metadata-model">The Metadata Model</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/what/mxe">Metadata Events</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/generated/metamodel/entities/dataplatform">Entities</a></div></li></ul></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/what-is-datahub/datahub-concepts">Concepts</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-standards">Metadata Standards</a><button aria-label="Toggle the collapsible sidebar category 'Metadata Standards'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/api/datahub-apis">APIs and SDKs Overview</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/api/graphql/overview">API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/as-a-library">Python SDK</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/metadata-integration/java/as-a-library">Java SDK</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/cli">DataHub CLI</a><button aria-label="Toggle the collapsible sidebar category 'DataHub CLI'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/act-on-metadata">DataHub Actions</a><button aria-label="Toggle the collapsible sidebar category 'DataHub Actions'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/api/tutorials/datasets">Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Admin</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authentication">Authentication</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authorization">Authorization</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/how/delete-metadata">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Deployment</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/deployment-guides">Deployment Guides</a><button aria-label="Toggle the collapsible sidebar category 'Deployment Guides'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/deploy/confluent-cloud">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Developers</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/architecture/architecture">Architecture</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/developers">Developing on DataHub</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/advanced/mcp-mcl">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Community</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/community">Community</a><button aria-label="Toggle the collapsible sidebar category 'Community'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/releases">Release History</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">DataHub's Open Metadata Standard</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">The Metadata Model</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>How does DataHub model metadata?</h1><p>DataHub takes a schema-first approach to modeling metadata. We use the open-source Pegasus schema language (<a href="https://linkedin.github.io/rest.li/pdl_schema" target="_blank" rel="noopener noreferrer">PDL</a>) extended with a custom set of annotations to model metadata. The DataHub storage, serving, indexing and ingestion layer operates directly on top of the metadata model and supports strong types all the way from the client to the storage layer.</p><p>Conceptually, metadata is modeled using the following abstractions</p><ul><li><p><strong>Entities</strong>: An entity is the primary node in the metadata graph. For example, an instance of a Dataset or a CorpUser is an Entity. An entity is made up of a type, e.g. 'dataset', a unique identifier (e.g. an 'urn') and groups of metadata attributes (e.g. documents) which we call aspects.</p></li><li><p><strong>Aspects</strong>: An aspect is a collection of attributes that describes a particular facet of an entity. They are the smallest atomic unit of write in DataHub. That is, multiple aspects associated with the same Entity can be updated independently. For example, DatasetProperties contains a collection of attributes that describes a Dataset. Aspects can be shared across entities, for example "Ownership" is an aspect that is re-used across all the Entities that have owners. Common aspects include</p><ul><li><a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl" target="_blank" rel="noopener noreferrer">ownership</a>: Captures the users and groups who own an Entity.</li><li><a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/GlobalTags.pdl" target="_blank" rel="noopener noreferrer">globalTags</a>: Captures references to the Tags associated with an Entity.</li><li><a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTerms.pdl" target="_blank" rel="noopener noreferrer">glossaryTerms</a>: Captures references to the Glossary Terms associated with an Entity.</li><li><a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/InstitutionalMemory.pdl" target="_blank" rel="noopener noreferrer">institutionalMemory</a>: Captures internal company Documents associated with an Entity (e.g. links!)</li><li><a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Status.pdl" target="_blank" rel="noopener noreferrer">status</a>: Captures the "deletion" status of an Entity, i.e. whether it should be soft-deleted.</li><li><a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/SubTypes.pdl" target="_blank" rel="noopener noreferrer">subTypes</a>: Captures one or more "sub types" of a more generic Entity type. An example can be a "Looker Explore" Dataset, a "View" Dataset. Specific sub types can imply that certain additional aspects are present for a given Entity.</li></ul></li><li><p><strong>Relationships</strong>: A relationship represents a named edge between 2 entities. They are declared via foreign key attributes within Aspects along with a custom annotation (@Relationship). Relationships permit edges to be traversed bi-directionally. For example, a Chart may refer to a CorpUser as its owner via a relationship named "OwnedBy". This edge would be walkable starting from the Chart <em>or</em> the CorpUser instance.</p></li><li><p><strong>Identifiers (Keys & Urns)</strong>: A key is a special type of aspect that contains the fields that uniquely identify an individual Entity. Key aspects can be serialized into <em>Urns</em>, which represent a stringified form of the key fields used for primary-key lookup. Moreover, <em>Urns</em> can be converted back into key aspect structs, making key aspects a type of "virtual" aspect. Key aspects provide a mechanism for clients to easily read fields comprising the primary key, which are usually generally useful like Dataset names, platform names etc. Urns provide a friendly handle by which Entities can be queried without requiring a fully materialized struct.</p></li></ul><p>Here is an example graph consisting of 3 types of entity (CorpUser, Chart, Dashboard), 2 types of relationship (OwnedBy, Contains), and 3 types of metadata aspect (Ownership, ChartInfo, and DashboardInfo).</p><p align="center"><img loading="lazy" width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/metadata-model-chart.png" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="the-core-entities">The Core Entities<a href="#the-core-entities" class="hash-link" aria-label="Direct link to The Core Entities" title="Direct link to The Core Entities"></a></h2><p>DataHub's "core" Entity types model the Data Assets that comprise the Modern Data Stack. They include</p><ol><li><strong><a href="/docs/generated/metamodel/entities/dataplatform">Data Platform</a></strong>: A type of Data "Platform". That is, an external system that is involved in processing, storing, or visualizing Data Assets. Examples include MySQL, Snowflake, Redshift, and S3.</li><li><strong><a href="/docs/generated/metamodel/entities/dataset">Dataset</a></strong>: A collection of data. Tables, Views, Streams, Document Collections, and Files are all modeled as "Datasets" on DataHub. Datasets can have tags, owners, links, glossary terms, and descriptions attached to them. They can also have specific sub-types, such as "View", "Collection", "Stream", "Explore", and more. Examples include Postgres Tables, MongoDB Collections, or S3 files.</li><li><strong><a href="/docs/generated/metamodel/entities/chart">Chart</a></strong>: A single data vizualization derived from a Dataset. A single Chart can be a part of multiple Dashboards. Charts can have tags, owners, links, glossary terms, and descriptions attached to them. Examples include a Superset or Looker Chart.</li><li><strong><a href="/docs/generated/metamodel/entities/dashboard">Dashboard</a></strong>: A collection of Charts for visualization. Dashboards can have tags, owners, links, glossary terms, and descriptions attached to them. Examples include a Superset or Mode Dashboard.</li><li><strong><a href="/docs/generated/metamodel/entities/datajob">Data Job</a></strong> (Task): An executable job that processes data assets, where "processing" implies consuming data, producing data, or both. Data Jobs can have tags, owners, links, glossary terms, and descriptions attached to them. They must belong to a single Data Flow. Examples include an Airflow Task.</li><li><strong><a href="/docs/generated/metamodel/entities/dataflow">Data Flow</a></strong> (Pipeline): An executable collection of Data Jobs with dependencies among them, or a DAG. Data Jobs can have tags, owners, links, glossary terms, and descriptions attached to them. Examples include an Airflow DAG.</li></ol><p>See the <strong>Metadata Modeling/Entities</strong> section on the left to explore the entire model.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="the-entity-registry">The Entity Registry<a href="#the-entity-registry" class="hash-link" aria-label="Direct link to The Entity Registry" title="Direct link to The Entity Registry"></a></h2><p>Where are Entities and their aspects defined in DataHub? Where does the Metadata Model "live"? The Metadata Model is stitched together by means
|
||
of an <strong>Entity Registry</strong>, a catalog of Entities that comprise the Metadata Graph along with the aspects associated with each. Put
|
||
simply, this is where the "schema" of the model is defined.</p><p>Traditionally, the Entity Registry was constructed using <a href="https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot" target="_blank" rel="noopener noreferrer">Snapshot</a> models, which are schemas that explicitly tie
|
||
an Entity to the Aspects associated with it. An example is <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/DatasetSnapshot.pdl" target="_blank" rel="noopener noreferrer">DatasetSnapshot</a>, which defines the core <code>Dataset</code> Entity.
|
||
The Aspects of the Dataset entity are captured via a union field inside a special "Aspect" schema. An example is
|
||
<a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/DatasetAspect.pdl" target="_blank" rel="noopener noreferrer">DatasetAspect</a>.
|
||
This file associates dataset-specific aspects (like <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProperties.pdl" target="_blank" rel="noopener noreferrer">DatasetProperties</a>) and common aspects (like <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl" target="_blank" rel="noopener noreferrer">Ownership</a>,
|
||
<a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/InstitutionalMemory.pdl" target="_blank" rel="noopener noreferrer">InstitutionalMemory</a>,
|
||
and <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Status.pdl" target="_blank" rel="noopener noreferrer">Status</a>)
|
||
to the Dataset Entity. This approach to defining Entities will soon be deprecated in favor of a new approach.</p><p>As of January 2022, DataHub has deprecated support for Snapshot models as a means of adding new entities. Instead,
|
||
the Entity Registry is defined inside a YAML configuration file called <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/resources/entity-registry.yml" target="_blank" rel="noopener noreferrer">entity-registry.yml</a>,
|
||
which is provided to DataHub's Metadata Service at start up. This file declares Entities and Aspects by referring to their <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl#L7" target="_blank" rel="noopener noreferrer">names</a>.
|
||
At boot time, DataHub validates the structure of the registry file and ensures that it can find PDL schemas associated with
|
||
each aspect name provided by configuration (via the <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl#L6" target="_blank" rel="noopener noreferrer">@Aspect</a> annotation).</p><p>By moving to this format, evolving the Metadata Model becomes much easier. Adding Entities & Aspects becomes a matter of adding a
|
||
to the YAML configuration, instead of creating new Snapshot / Aspect files.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="exploring-datahubs-metadata-model">Exploring DataHub's Metadata Model<a href="#exploring-datahubs-metadata-model" class="hash-link" aria-label="Direct link to Exploring DataHub's Metadata Model" title="Direct link to Exploring DataHub's Metadata Model"></a></h2><p>To explore the current DataHub metadata model, you can inspect this high-level picture that shows the different entities and edges between them showing the relationships between them.</p><p align="center"><img loading="lazy" width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-metadata-model.png" class="img_ev3q"></p><p>To navigate the aspect model for specific entities and explore relationships using the <code>foreign-key</code> concept, you can view them in our demo environment or navigate the auto-generated docs in the <strong>Metadata Modeling/Entities</strong> section on the left.</p><p>For example, here are helpful links to the most popular entities in DataHub's metadata model:</p><ul><li><a href="/docs/generated/metamodel/entities/dataset">Dataset</a>: <a href="https://demo.datahub.com/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,Dataset,PROD)/Schema?is_lineage_mode=false" target="_blank" rel="noopener noreferrer">Profile</a> <a href="https://demo.datahub.com/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,Dataset,PROD)/Documentation?is_lineage_mode=false" target="_blank" rel="noopener noreferrer">Documentation</a></li><li><a href="/docs/generated/metamodel/entities/dashboard">Dashboard</a>: <a href="https://demo.datahub.com/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,Dashboard,PROD)/Schema?is_lineage_mode=false" target="_blank" rel="noopener noreferrer">Profile</a> <a href="https://demo.datahub.com/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,Dashboard,PROD)/Documentation?is_lineage_mode=false" target="_blank" rel="noopener noreferrer">Documentation</a></li><li><a href="/docs/generated/metamodel/entities/corpuser">User (a.k.a CorpUser)</a>: <a href="https://demo.datahub.com/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,Corpuser,PROD)/Schema?is_lineage_mode=false" target="_blank" rel="noopener noreferrer">Profile</a> <a href="https://demo.datahub.com/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,Corpuser,PROD)/Documentation?is_lineage_mode=false" target="_blank" rel="noopener noreferrer">Documentation</a></li><li><a href="/docs/generated/metamodel/entities/dataflow">Pipeline (a.k.a DataFlow)</a>: <a href="https://demo.datahub.com/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,DataFlow,PROD)/Schema?is_lineage_mode=false" target="_blank" rel="noopener noreferrer">Profile</a> <a href="https://demo.datahub.com/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,DataFlow,PROD)/Documentation?is_lineage_mode=false" target="_blank" rel="noopener noreferrer">Documentation</a></li><li><a href="/docs/generated/metamodel/entities/mlfeaturetable">Feature Table (a.k.a. MLFeatureTable)</a>: <a href="https://demo.datahub.com/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,MlFeatureTable,PROD)/Schema?is_lineage_mode=false" target="_blank" rel="noopener noreferrer">Profile</a> <a href="https://demo.datahub.com/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,MlFeatureTable,PROD)/Documentation?is_lineage_mode=false" target="_blank" rel="noopener noreferrer">Documentation</a></li><li>For the full list of entities in the metadata model, browse them <a href="https://demo.datahub.com/browse/dataset/prod/datahub/entities" target="_blank" rel="noopener noreferrer">here</a> or use the <strong>Metadata Modeling/Entities</strong> section on the left.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generating-documentation-for-the-metadata-model">Generating documentation for the Metadata Model<a href="#generating-documentation-for-the-metadata-model" class="hash-link" aria-label="Direct link to Generating documentation for the Metadata Model" title="Direct link to Generating documentation for the Metadata Model"></a></h3><ul><li>This website: Metadata model documentation for this website is generated using <code>./gradlew :docs-website:yarnBuild</code>, which delegates the model doc generation to the <code>modelDocGen</code> task in the <code>metadata-ingestion</code> module.</li><li>Uploading documentation to a running DataHub Instance: The metadata model documentation can be generated and uploaded into a running DataHub instance using the command <code>./gradlew :metadata-ingestion:modelDocUpload</code>. <strong><em>NOTE</em></strong>: This will upload the model documentation to the DataHub instance running at the environment variable <code>$DATAHUB_SERVER</code> (http://localhost:8080 by default)</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="querying-the-metadata-graph">Querying the Metadata Graph<a href="#querying-the-metadata-graph" class="hash-link" aria-label="Direct link to Querying the Metadata Graph" title="Direct link to Querying the Metadata Graph"></a></h2><p>DataHub’s modeling language allows you to optimize metadata persistence to align with query patterns.</p><p>There are three supported ways to query the metadata graph: by primary key lookup, a search query, and via relationship traversal.</p><blockquote><p>New to <a href="https://linkedin.github.io/rest.li/pdl_schema" target="_blank" rel="noopener noreferrer">PDL</a> files? Don't fret. They are just a way to define a JSON document "schema" for Aspects in DataHub. All Data ingested to DataHub's Metadata Service is validated against a PDL schema, with each @Aspect corresponding to a single schema. Structurally, PDL is quite similar to <a href="https://developers.google.com/protocol-buffers" target="_blank" rel="noopener noreferrer">Protobuf</a> and conveniently maps to JSON.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="querying-an-entity">Querying an Entity<a href="#querying-an-entity" class="hash-link" aria-label="Direct link to Querying an Entity" title="Direct link to Querying an Entity"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="fetching-latest-entity-aspects-snapshot">Fetching Latest Entity Aspects (Snapshot)<a href="#fetching-latest-entity-aspects-snapshot" class="hash-link" aria-label="Direct link to Fetching Latest Entity Aspects (Snapshot)" title="Direct link to Fetching Latest Entity Aspects (Snapshot)"></a></h4><p>Querying an Entity by primary key means using the "entities" endpoint, passing in the
|
||
urn of the entity to retrieve.</p><p>For example, to fetch a Chart entity, we can use the following <code>curl</code>:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">curl --location --request GET 'http://localhost:8080/entities/urn%3Ali%3Achart%3Acustomers</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This request will return a set of versioned aspects, each at the latest version.</p><p>As you'll notice, we perform the lookup using the url-encoded <em>Urn</em> associated with an entity.
|
||
The response would be an "Entity" record containing the Entity Snapshot (which in turn contains the latest aspects associated with the Entity).</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="fetching-versioned-aspects">Fetching Versioned Aspects<a href="#fetching-versioned-aspects" class="hash-link" aria-label="Direct link to Fetching Versioned Aspects" title="Direct link to Fetching Versioned Aspects"></a></h4><p>DataHub also supports fetching individual pieces of metadata about an Entity, which we call aspects. To do so,
|
||
you'll provide both an Entity's primary key (urn) along with the aspect name and version that you'd like to retrieve.</p><p>For example, to fetch the latest version of a Dataset's SchemaMetadata aspect, you would issue the following query:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">curl 'http://localhost:8080/aspects/urn%3Ali%3Adataset%3A(urn%3Ali%3AdataPlatform%3Afoo%2Cbar%2CPROD)?aspect=schemaMetadata&version=0'</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "version":0,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "aspect":{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "com.linkedin.schema.SchemaMetadata":{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "created":{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "actor":"urn:li:corpuser:fbar",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "time":0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "platformSchema":{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "com.linkedin.schema.KafkaSchema":{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "documentSchema":"{\"type\":\"record\",\"name\":\"MetadataChangeEvent\",\"namespace\":\"com.linkedin.mxe\",\"doc\":\"Kafka event for proposing a metadata change for an entity.\",\"fields\":[{\"name\":\"auditHeader\",\"type\":{\"type\":\"record\",\"name\":\"KafkaAuditHeader\",\"namespace\":\"com.linkedin.avro2pegasus.events\",\"doc\":\"Header\"}}]}"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "lastModified":{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "actor":"urn:li:corpuser:fbar",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "time":0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "schemaName":"FooEvent",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "fields":[</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "fieldPath":"foo",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "description":"Bar",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type":{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type":{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "com.linkedin.schema.StringType":{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "nativeDataType":"string"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ],</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "version":0,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "hash":"",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "platform":"urn:li:dataPlatform:foo"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="fetching-timeseries-aspects">Fetching Timeseries Aspects<a href="#fetching-timeseries-aspects" class="hash-link" aria-label="Direct link to Fetching Timeseries Aspects" title="Direct link to Fetching Timeseries Aspects"></a></h4><p>DataHub supports an API for fetching a group of Timeseries aspects about an Entity. For example, you may want to use this API
|
||
to fetch recent profiling runs & statistics about a Dataset. To do so, you can issue a "get" request against the <code>/aspects</code> endpoint.</p><p>For example, to fetch dataset profiles (ie. stats) for a Dataset, you would issue the following query:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">curl -X POST 'http://localhost:8080/aspects?action=getTimeseriesAspectValues' \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--data '{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "urn": "urn:li:dataset:(urn:li:dataPlatform:redshift,global_dev.larxynx_carcinoma_data_2020,PROD)",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "entity": "dataset",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "aspect": "datasetProfile",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "startTimeMillis": 1625122800000,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "endTimeMillis": 1627455600000</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}'</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "value":{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "limit":2000,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "aspectName":"datasetProfile",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "endTimeMillis":1627455600000,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "startTimeMillis":1625122800000,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "entityName":"dataset",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "values":[</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "aspect":{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "value":"{\"timestampMillis\":1626912000000,\"fieldProfiles\":[{\"uniqueProportion\":1.0,\"sampleValues\":[\"123MMKK12\",\"13KDFMKML\",\"123NNJJJL\"],\"fieldPath\":\"id\",\"nullCount\":0,\"nullProportion\":0.0,\"uniqueCount\":3742},{\"uniqueProportion\":1.0,\"min\":\"1524406400000\",\"max\":\"1624406400000\",\"sampleValues\":[\"1640023230002\",\"1640343012207\",\"16303412330117\"],\"mean\":\"1555406400000\",\"fieldPath\":\"date\",\"nullCount\":0,\"nullProportion\":0.0,\"uniqueCount\":3742},{\"uniqueProportion\":0.037,\"min\":\"21\",\"median\":\"68\",\"max\":\"92\",\"sampleValues\":[\"45\",\"65\",\"81\"],\"mean\":\"65\",\"distinctValueFrequencies\":[{\"value\":\"12\",\"frequency\":103},{\"value\":\"54\",\"frequency\":12}],\"fieldPath\":\"patient_age\",\"nullCount\":0,\"nullProportion\":0.0,\"uniqueCount\":79},{\"uniqueProportion\":0.00820873786407767,\"sampleValues\":[\"male\",\"female\"],\"fieldPath\":\"patient_gender\",\"nullCount\":120,\"nullProportion\":0.03,\"uniqueCount\":2}],\"rowCount\":3742,\"columnCount\":4}",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "contentType":"application/json"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You'll notice that the aspect itself is serialized as escaped JSON. This is part of a shift toward a more generic set of READ / WRITE APIs
|
||
that permit serialization of aspects in different ways. By default, the content type will be JSON, and the aspect can be deserialized into a normal JSON object
|
||
in the language of your choice. Note that this will soon become the de-facto way to both write and read individual aspects.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="search-query">Search Query<a href="#search-query" class="hash-link" aria-label="Direct link to Search Query" title="Direct link to Search Query"></a></h3><p>A search query allows you to search for entities matching an arbitrary string.</p><p>For example, to search for entities matching the term "customers", we can use the following CURL:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">curl --location --request POST 'http://localhost:8080/entities?action=search' \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--header 'X-RestLi-Protocol-Version: 2.0.0' \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--header 'Content-Type: application/json' \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--data-raw '{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "input": "\"customers\"",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "entity": "chart",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "start": 0,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "count": 10</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}'</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The notable parameters are <code>input</code> and <code>entity</code>. <code>input</code> specifies the query we are issuing and <code>entity</code> specifies the Entity Type we want to search over. This is the common name of the Entity as defined in the @Entity definition. The response contains a list of Urns, that can be used to fetch the full entity.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="relationship-query">Relationship Query<a href="#relationship-query" class="hash-link" aria-label="Direct link to Relationship Query" title="Direct link to Relationship Query"></a></h3><p>A relationship query allows you to find Entity connected to a particular source Entity via an edge of a particular type.</p><p>For example, to find the owners of a particular Chart, we can use the following CURL:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">curl --location --request GET --header 'X-RestLi-Protocol-Version: 2.0.0' 'http://localhost:8080/relationships?direction=OUTGOING&urn=urn%3Ali%3Achart%3Acustomers&types=List(OwnedBy)'</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The notable parameters are <code>direction</code>, <code>urn</code> and <code>types</code>. The response contains <em>Urns</em> associated with all entities connected
|
||
to the primary entity (urn:li:chart:customer) by an relationship named "OwnedBy". That is, it permits fetching the owners of a given
|
||
chart.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="special-aspects">Special Aspects<a href="#special-aspects" class="hash-link" aria-label="Direct link to Special Aspects" title="Direct link to Special Aspects"></a></h3><p>There are a few special aspects worth mentioning:</p><ol><li>Key aspects: Contain the properties that uniquely identify an Entity.</li><li>Browse Paths aspect: Represents a hierarchical path associated with an Entity.</li></ol><h4 class="anchor anchorWithStickyNavbar_LWe7" id="key-aspects">Key aspects<a href="#key-aspects" class="hash-link" aria-label="Direct link to Key aspects" title="Direct link to Key aspects"></a></h4><p>As introduced above, Key aspects are structs / records that contain the fields that uniquely identify an Entity. There are
|
||
some constraints about the fields that can be present in Key aspects:</p><ul><li>All fields must be of STRING or ENUM type</li><li>All fields must be REQUIRED</li></ul><p>Keys can be created from and turned into <em>Urns</em>, which represent the stringified version of the Key record.
|
||
The algorithm used to do the conversion is straightforward: the fields of the Key aspect are substituted into a
|
||
string template based on their index (order of definition) using the following template:</p><div class="language-aidl codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-aidl codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">// Case 1: # key fields == 1</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">urn:li:<entity-name>:key-field-1</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">// Case 2: # key fields > 1</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">urn:li:<entity-name>:(key-field-1, key-field-2, ... key-field-n)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>By convention, key aspects are defined under <a href="https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin/metadata/key" target="_blank" rel="noopener noreferrer">metadata-models/src/main/pegasus/com/linkedin/metadata/key</a>.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="example">Example<a href="#example" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example"></a></h5><p>A CorpUser can be uniquely identified by a "username", which should typically correspond to an LDAP name.</p><p>Thus, it's Key Aspect is defined as the following:</p><div class="language-aidl codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-aidl codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">namespace com.linkedin.metadata.key</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">/**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * Key for a CorpUser</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">@Aspect = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": "corpUserKey"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">record CorpUserKey {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * The name of the AD/LDAP user.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> username: string</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>and it's Entity Snapshot model is defined as</p><div class="language-aidl codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-aidl codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">/**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * A metadata snapshot for a specific CorpUser entity.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">@Entity = {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": "corpuser",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "keyAspect": "corpUserKey"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">record CorpUserSnapshot {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * URN for the entity the metadata snapshot is associated with.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> urn: CorpuserUrn</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> * The list of metadata aspects associated with the CorpUser. Depending on the use case, this can either be all, or a selection, of supported aspects.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> aspects: array[CorpUserAspect]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Using a combination of the information provided by these models, we are able to generate the Urn corresponding to a CorpUser as</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">urn:li:corpuser:<username></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Imagine we have a CorpUser Entity with the username "johnsmith". In this world, the JSON version of the Key Aspect associated with the Entity would be</p><div class="language-aidl codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-aidl codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "username": "johnsmith"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>and its corresponding Urn would be</p><div class="language-aidl codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-aidl codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">urn:li:corpuser:johnsmith</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="browsepaths-aspect">BrowsePaths aspect<a href="#browsepaths-aspect" class="hash-link" aria-label="Direct link to BrowsePaths aspect" title="Direct link to BrowsePaths aspect"></a></h4><p>The BrowsePaths aspect allows you to define a custom "browse path" for an Entity. A browse path is a way to hierarchically organize
|
||
entities. They manifest within the "Explore" features on the UI, allowing users to navigate through trees of related entities of a given type.</p><p>To support browsing a particular entity, add the "browsePaths" aspect to the entity in your <code>entity-registry.yml</code> file.</p><div class="language-aidl codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-aidl codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">/// entity-registry.yml</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">entities:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> - name: dataset</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> doc: Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> keyAspect: datasetKey</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> aspects:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> - browsePaths</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>By declaring this aspect, you can produce custom browse paths as well as query for browse paths manually using a CURL like the following:</p><div class="language-aidl codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-aidl codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">curl --location --request POST 'http://localhost:8080/entities?action=browse' \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--header 'X-RestLi-Protocol-Version: 2.0.0' \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--header 'Content-Type: application/json' \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--data-raw '{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "path": "/my/custom/browse/path",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "entity": "dataset",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "start": 0,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "limit": 10</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}'</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Please note you must provide:</p><ul><li>The "/"-delimited root path for which to fetch results.</li><li>An entity "type" using its common name ("dataset" in the example above).</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="types-of-aspect">Types of Aspect<a href="#types-of-aspect" class="hash-link" aria-label="Direct link to Types of Aspect" title="Direct link to Types of Aspect"></a></h3><p>There are 2 "types" of Metadata Aspects. Both are modeled using PDL schemas, and both can be ingested in the same way.
|
||
However, they differ in what they represent and how they are handled by DataHub's Metadata Service.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="1-versioned-aspects">1. Versioned Aspects<a href="#1-versioned-aspects" class="hash-link" aria-label="Direct link to 1. Versioned Aspects" title="Direct link to 1. Versioned Aspects"></a></h4><p>Versioned Aspects each have a <strong>numeric version</strong> associated with them. When a field in an aspect changes, a new
|
||
version is automatically created and stored within DataHub's backend. In practice, all versioned aspects are stored inside a relational database
|
||
that can be backed up and restored. Versioned aspects power much of the UI experience you're used to, including Ownership, Descriptions,
|
||
Tags, Glossary Terms, and more. Examples include Ownership, Global Tags, and Glossary Terms.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="2-timeseries-aspects">2. Timeseries Aspects<a href="#2-timeseries-aspects" class="hash-link" aria-label="Direct link to 2. Timeseries Aspects" title="Direct link to 2. Timeseries Aspects"></a></h4><p>Timeseries Aspects each have a <strong>timestamp</strong> associated with them. They are useful for representing
|
||
time-ordered events about an Entity. For example, the results of profiling a Dataset, or a set of Data Quality checks that
|
||
run every day. It is important to note that Timeseries aspects are NOT persisted inside the relational store, and are instead
|
||
persisted only in the search index (e.g. elasticsearch) and the message queue (Kafka). This makes restoring timeseries aspects
|
||
in a disaster scenario a bit more challenge. Timeseries aspects can be queried by time range, which is what makes them most different from Versioned Aspects.
|
||
A timeseries aspect can be identified by the "timeseries" <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProfile.pdl#L10" target="_blank" rel="noopener noreferrer">type</a> in its <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProfile.pdl#L8" target="_blank" rel="noopener noreferrer">@Aspect</a> annotation.
|
||
Examples include <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProfile.pdl" target="_blank" rel="noopener noreferrer">DatasetProfile</a> & <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetUsageStatistics.pdl" target="_blank" rel="noopener noreferrer">DatasetUsageStatistics</a>.</p><p>Timeseries aspects are aspects that have a timestampMillis field, and are meant for aspects that continuously change on a
|
||
timely basis e.g. data profiles, usage statistics, etc.</p><p>Each timeseries aspect must be declared "type": "timeseries" and must
|
||
include <a href="https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin/timeseries/TimeseriesAspectBase.pdl" target="_blank" rel="noopener noreferrer">TimeseriesAspectBase</a>
|
||
, which contains a timestampMillis field.</p><p>Timeseries aspect can also have fields annotated with @Searchable and @Relationship.</p><p>Please refer
|
||
to <a href="https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProfile.pdl" target="_blank" rel="noopener noreferrer">DatasetProfile</a>
|
||
to see an example of a timeseries aspect.</p><p>Because timeseries aspects are updated on a frequent basis, ingests of these aspects go straight to elastic search (
|
||
instead of being stored in local DB).</p><p>You can retrieve timeseries aspects using the "aspects?action=getTimeseriesAspectValues" end point.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="aggregatable-timeseries-aspects">Aggregatable Timeseries aspects<a href="#aggregatable-timeseries-aspects" class="hash-link" aria-label="Direct link to Aggregatable Timeseries aspects" title="Direct link to Aggregatable Timeseries aspects"></a></h5><p>Being able to perform SQL like <em>group by + aggregate</em> operations on the timeseries aspects is a very natural use-case for
|
||
this kind of data (dataset profiles, usage statistics etc.). This section describes how to define, ingest and perform an
|
||
aggregation query against a timeseries aspect.</p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="defining-a-new-aggregatable-timeseries-aspect">Defining a new aggregatable Timeseries aspect.<a href="#defining-a-new-aggregatable-timeseries-aspect" class="hash-link" aria-label="Direct link to Defining a new aggregatable Timeseries aspect." title="Direct link to Defining a new aggregatable Timeseries aspect."></a></h6><p>The <em>@TimeseriesField</em> and the <em>@TimeseriesFieldCollection</em> are two new annotations that can be attached to a field of
|
||
a <em>Timeseries aspect</em> that allows it to be part of an aggregatable query. The kinds of aggregations allowed on these
|
||
annotated fields depends on the type of the field, as well as the kind of aggregation, as
|
||
described <a href="#performing-an-aggregation-on-a-timeseries-aspect">here</a>.</p><ul><li><p><code>@TimeseriesField = {}</code> - this annotation can be used with any type of non-collection type field of the aspect such as
|
||
primitive types and records (see the fields <em>stat</em>, <em>strStat</em> and <em>strArray</em> fields
|
||
of <a href="https://github.com/datahub-project/datahub/blob/master/test-models/src/main/pegasus/com/datahub/test/TestEntityProfile.pdl" target="_blank" rel="noopener noreferrer">TestEntityProfile.pdl</a>).</p></li><li><p>The <code>@TimeseriesFieldCollection {"key":"<name of the key field of collection item type>"}</code> annotation allows for
|
||
aggregation support on the items of a collection type (supported only for the array type collections for now), where the
|
||
value of <code>"key"</code> is the name of the field in the collection item type that will be used to specify the group-by clause (
|
||
see <em>userCounts</em> and <em>fieldCounts</em> fields of <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetUsageStatistics.pdl" target="_blank" rel="noopener noreferrer">DatasetUsageStatistics.pdl</a>).</p></li></ul><p>In addition to defining the new aspect with appropriate Timeseries annotations,
|
||
the <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/resources/entity-registry.yml" target="_blank" rel="noopener noreferrer">entity-registry.yml</a>
|
||
file needs to be updated as well. Just add the new aspect name under the list of aspects against the appropriate entity as shown below, such as <code>datasetUsageStatistics</code> for the aspect DatasetUsageStatistics.</p><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token key atrule">entities</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> dataset</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">keyAspect</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> datasetKey</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">aspects</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> datasetProfile</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> datasetUsageStatistics</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h6 class="anchor anchorWithStickyNavbar_LWe7" id="ingesting-a-timeseries-aspect">Ingesting a Timeseries aspect<a href="#ingesting-a-timeseries-aspect" class="hash-link" aria-label="Direct link to Ingesting a Timeseries aspect" title="Direct link to Ingesting a Timeseries aspect"></a></h6><p>The timeseries aspects can be ingested via the GMS REST endpoint <code>/aspects?action=ingestProposal</code> or via the python API.</p><p>Example1: Via GMS REST API using curl.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token function" style="color:rgb(130, 170, 255)">curl</span><span class="token plain"> --location --request POST </span><span class="token string" style="color:rgb(195, 232, 141)">'http://localhost:8080/aspects?action=ingestProposal'</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--header </span><span class="token string" style="color:rgb(195, 232, 141)">'X-RestLi-Protocol-Version: 2.0.0'</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--header </span><span class="token string" style="color:rgb(195, 232, 141)">'Content-Type: application/json'</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--data-raw </span><span class="token string" style="color:rgb(195, 232, 141)">'{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "proposal" : {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "entityType": "dataset",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "entityUrn" : "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "changeType" : "UPSERT",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "aspectName" : "datasetUsageStatistics",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "aspect" : {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "value" : "{ \"timestampMillis\":1629840771000,\"uniqueUserCount\" : 10, \"totalSqlQueries\": 20, \"fieldCounts\": [ {\"fieldPath\": \"col1\", \"count\": 20}, {\"fieldPath\" : \"col2\", \"count\": 5} ]}",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "contentType": "application/json"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)">}'</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Example2: Via Python API to Kafka(or REST)</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">from</span><span class="token plain"> datahub</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">metadata</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">schema_classes </span><span class="token keyword" style="font-style:italic">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ChangeTypeClass</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> DatasetFieldUsageCountsClass</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> DatasetUsageStatisticsClass</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token keyword" style="font-style:italic">from</span><span class="token plain"> datahub</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">emitter</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">kafka_emitter </span><span class="token keyword" style="font-style:italic">import</span><span class="token plain"> DatahubKafkaEmitter</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token keyword" style="font-style:italic">from</span><span class="token plain"> datahub</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">emitter</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">mcp </span><span class="token keyword" style="font-style:italic">import</span><span class="token plain"> MetadataChangeProposalWrapper</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token keyword" style="font-style:italic">from</span><span class="token plain"> datahub</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">emitter</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">rest_emitter </span><span class="token keyword" style="font-style:italic">import</span><span class="token plain"> DatahubRestEmitter</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">usageStats </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> DatasetUsageStatisticsClass</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> timestampMillis</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">1629840771000</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> uniqueUserCount</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">10</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> totalSqlQueries</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">20</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> fieldCounts</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> DatasetFieldUsageCountsClass</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> fieldPath</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token string" style="color:rgb(195, 232, 141)">"col1"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> count</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">mcpw </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> MetadataChangeProposalWrapper</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> entityType</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token string" style="color:rgb(195, 232, 141)">"dataset"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> aspectName</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token string" style="color:rgb(195, 232, 141)">"datasetUsageStatistics"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> changeType</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">ChangeTypeClass</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">UPSERT</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> entityUrn</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> aspect</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">usageStats</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Instantiate appropriate emitter (kafka_emitter/rest_emitter)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># my_emitter = DatahubKafkaEmitter("""<config>""")</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">my_emitter </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> DatahubRestEmitter</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token string" style="color:rgb(195, 232, 141)">"http://localhost:8080"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">my_emitter</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">emit</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">mcpw</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h6 class="anchor anchorWithStickyNavbar_LWe7" id="performing-an-aggregation-on-a-timeseries-aspect">Performing an aggregation on a Timeseries aspect<a href="#performing-an-aggregation-on-a-timeseries-aspect" class="hash-link" aria-label="Direct link to Performing an aggregation on a Timeseries aspect" title="Direct link to Performing an aggregation on a Timeseries aspect"></a></h6><p>Aggreations on timeseries aspects can be performed by the GMS REST API for <code>/analytics?action=getTimeseriesStats</code> which
|
||
accepts the following params.</p><ul><li><code>entityName</code> - The name of the entity the aspect is associated with.</li><li><code>aspectName</code> - The name of the aspect.</li><li><code>filter</code> - Any pre-filtering criteria before grouping and aggregations are performed.</li><li><code>metrics</code> - A list of aggregation specification. The <code>fieldPath</code> member of an aggregation specification refers to the
|
||
field name against which the aggregation needs to be performed, and the <code>aggregationType</code> specifies the kind of aggregation.</li><li><code>buckets</code> - A list of grouping bucket specifications. Each grouping bucket has a <code>key</code> field that refers to the field
|
||
to use for grouping. The <code>type</code> field specifies the kind of grouping bucket.</li></ul><p>We support three kinds of aggregations that can be specified in an aggregation query on the Timeseries annotated fields.
|
||
The values that <code>aggregationType</code> can take are:</p><ul><li><code>LATEST</code>: The latest value of the field in each bucket. Supported for any type of field.</li><li><code>SUM</code>: The cumulative sum of the field in each bucket. Supported only for integral types.</li><li><code>CARDINALITY</code>: The number of unique values or the cardinality of the set in each bucket. Supported for string and
|
||
record types.</li></ul><p>We support two types of grouping for defining the buckets to perform aggregations against:</p><ul><li><code>DATE_GROUPING_BUCKET</code>: Allows for creating time-based buckets such as by second, minute, hour, day, week, month,
|
||
quarter, year etc. Should be used in conjunction with a timestamp field whose value is in milliseconds since <em>epoch</em>.
|
||
The <code>timeWindowSize</code> param specifies the date histogram bucket width.</li><li><code>STRING_GROUPING_BUCKET</code>: Allows for creating buckets grouped by the unique values of a field. Should always be used in
|
||
conjunction with a string type field.</li></ul><p>The API returns a generic SQL like table as the <code>table</code> member of the output that contains the results of
|
||
the <code>group-by/aggregate</code> query, in addition to echoing the input params.</p><ul><li><code>columnNames</code>: the names of the table columns. The group-by <code>key</code> names appear in the same order as they are specified
|
||
in the request. Aggregation specifications follow the grouping fields in the same order as specified in the request,
|
||
and will be named <code><agg_name>_<fieldPath></code>.</li><li><code>columnTypes</code>: the data types of the columns.</li><li><code>rows</code>: the data values, each row corresponding to the respective bucket(s).</li></ul><p>Example: Latest unique user count for each day.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># QUERY</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token function" style="color:rgb(130, 170, 255)">curl</span><span class="token plain"> --location --request POST </span><span class="token string" style="color:rgb(195, 232, 141)">'http://localhost:8080/analytics?action=getTimeseriesStats'</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--header </span><span class="token string" style="color:rgb(195, 232, 141)">'X-RestLi-Protocol-Version: 2.0.0'</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--header </span><span class="token string" style="color:rgb(195, 232, 141)">'Content-Type: application/json'</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--data-raw </span><span class="token string" style="color:rgb(195, 232, 141)">'{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "entityName": "dataset",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "aspectName": "datasetUsageStatistics",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "filter": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "criteria": []</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "metrics": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "fieldPath": "uniqueUserCount",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "aggregationType": "LATEST"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> ],</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "buckets": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "key": "timestampMillis",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "type": "DATE_GROUPING_BUCKET",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "timeWindowSize": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "multiple": 1,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "unit": "DAY"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)">}'</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># SAMPLE RESPOSNE</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"value"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"filter"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"criteria"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"aspectName"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"datasetUsageStatistics"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"entityName"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"dataset"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"groupingBuckets"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"type"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"DATE_GROUPING_BUCKET"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"timeWindowSize"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"multiple"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"unit"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"DAY"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"key"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"timestampMillis"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"aggregationSpecs"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"fieldPath"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"uniqueUserCount"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"aggregationType"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"LATEST"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"table"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"columnNames"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"timestampMillis"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"latest_uniqueUserCount"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"rows"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"1631491200000"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"1"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"columnTypes"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"long"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"int"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For more examples on the complex types of group-by/aggregations, refer to the tests in the group <code>getAggregatedStats</code> of <a href="https://github.com/datahub-project/datahub/blob/master/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java" target="_blank" rel="noopener noreferrer">TimeseriesAspectServiceTestBase.java</a>.</p></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="slackUtm_uoBr"><div class="slackUtm_uoBr"><hr>Need more help? Join the conversation in <a href="https://datahub.com/slack?utm_source=docs&utm_medium=footer&utm_campaign=docs_footer&utm_content=docs/modeling/metadata-model">Slack!</a></div></div><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/datahub-project/datahub/blob/master/docs/modeling/metadata-model.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_VsjB"></div></div></footer><div class="feedbackWrapper_mUHF"><div class="feedbackWidget_PX4d"><div class="feedbackButtons_wn3V"><strong>Is this page helpful?</strong><div><button class="feedbackButton_UgQs"><span role="img" aria-label="like" class="anticon anticon-like"><svg viewBox="64 64 896 896" focusable="false" data-icon="like" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M885.9 533.7c16.8-22.2 26.1-49.4 26.1-77.7 0-44.9-25.1-87.4-65.5-111.1a67.67 67.67 0 00-34.3-9.3H572.4l6-122.9c1.4-29.7-9.1-57.9-29.5-79.4A106.62 106.62 0 00471 99.9c-52 0-98 35-111.8 85.1l-85.9 311H144c-17.7 0-32 14.3-32 32v364c0 17.7 14.3 32 32 32h601.3c9.2 0 18.2-1.8 26.5-5.4 47.6-20.3 78.3-66.8 78.3-118.4 0-12.6-1.8-25-5.4-37 16.8-22.2 26.1-49.4 26.1-77.7 0-12.6-1.8-25-5.4-37 16.8-22.2 26.1-49.4 26.1-77.7-.2-12.6-2-25.1-5.6-37.1zM184 852V568h81v284h-81zm636.4-353l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 16.5-7.2 32.2-19.6 43l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 16.5-7.2 32.2-19.6 43l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 22.4-13.2 42.6-33.6 51.8H329V564.8l99.5-360.5a44.1 44.1 0 0142.2-32.3c7.6 0 15.1 2.2 21.1 6.7 9.9 7.4 15.2 18.6 14.6 30.5l-9.6 198.4h314.4C829 418.5 840 436.9 840 456c0 16.5-7.2 32.1-19.6 43z"></path></svg></span></button><button class="feedbackButton_UgQs"><span role="img" aria-label="dislike" class="anticon anticon-dislike"><svg viewBox="64 64 896 896" focusable="false" data-icon="dislike" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M885.9 490.3c3.6-12 5.4-24.4 5.4-37 0-28.3-9.3-55.5-26.1-77.7 3.6-12 5.4-24.4 5.4-37 0-28.3-9.3-55.5-26.1-77.7 3.6-12 5.4-24.4 5.4-37 0-51.6-30.7-98.1-78.3-118.4a66.1 66.1 0 00-26.5-5.4H144c-17.7 0-32 14.3-32 32v364c0 17.7 14.3 32 32 32h129.3l85.8 310.8C372.9 889 418.9 924 470.9 924c29.7 0 57.4-11.8 77.9-33.4 20.5-21.5 31-49.7 29.5-79.4l-6-122.9h239.9c12.1 0 23.9-3.2 34.3-9.3 40.4-23.5 65.5-66.1 65.5-111 0-28.3-9.3-55.5-26.1-77.7zM184 456V172h81v284h-81zm627.2 160.4H496.8l9.6 198.4c.6 11.9-4.7 23.1-14.6 30.5-6.1 4.5-13.6 6.8-21.1 6.7a44.28 44.28 0 01-42.2-32.3L329 459.2V172h415.4a56.85 56.85 0 0133.6 51.8c0 9.7-2.3 18.9-6.9 27.3l-13.9 25.4 21.9 19a56.76 56.76 0 0119.6 43c0 9.7-2.3 18.9-6.9 27.3l-13.9 25.4 21.9 19a56.76 56.76 0 0119.6 43c0 9.7-2.3 18.9-6.9 27.3l-14 25.5 21.9 19a56.76 56.76 0 0119.6 43c0 19.1-11 37.5-28.8 48.4z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/metadata-ingestion/docs/dev_guides/profiling_ingestions"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Profiling ingestions</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/what/mxe"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Metadata Events</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#the-core-entities" class="table-of-contents__link toc-highlight">The Core Entities</a></li><li><a href="#the-entity-registry" class="table-of-contents__link toc-highlight">The Entity Registry</a></li><li><a href="#exploring-datahubs-metadata-model" class="table-of-contents__link toc-highlight">Exploring DataHub's Metadata Model</a><ul><li><a href="#generating-documentation-for-the-metadata-model" class="table-of-contents__link toc-highlight">Generating documentation for the Metadata Model</a></li></ul></li><li><a href="#querying-the-metadata-graph" class="table-of-contents__link toc-highlight">Querying the Metadata Graph</a><ul><li><a href="#querying-an-entity" class="table-of-contents__link toc-highlight">Querying an Entity</a></li><li><a href="#search-query" class="table-of-contents__link toc-highlight">Search Query</a></li><li><a href="#relationship-query" class="table-of-contents__link toc-highlight">Relationship Query</a></li><li><a href="#special-aspects" class="table-of-contents__link toc-highlight">Special Aspects</a></li><li><a href="#types-of-aspect" class="table-of-contents__link toc-highlight">Types of Aspect</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/docs/">Introduction</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/quickstart">Quickstart</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://datahub.com/slack" target="_blank" rel="noopener noreferrer" class="footer__link-item">Slack<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="footer__link-item">YouTube<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="footer__link-item">Blog<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/docs/townhalls">Town Halls</a></li><li class="footer__item"><a href="https://datahub.com/adoption-stories/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Adoption<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://demo.datahub.com/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Demo</a></li><li class="footer__item"><a href="https://feature-requests.datahubproject.io/roadmap" target="_blank" rel="noopener noreferrer" class="footer__link-item">Roadmap<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/docs/contributing">Contributing</a></li><li class="footer__item"><a href="https://github.com/datahub-project/datahub" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://feature-requests.datahubproject.io/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Feature Requests<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2015-2025 DataHub Project Authors.</div></div></div></footer></div>
|
||
<script src="/assets/js/runtime~main.50e13f51.js"></script>
|
||
<script src="/assets/js/main.edc0853c.js"></script>
|
||
</body>
|
||
</html> |