mirror of
https://github.com/datahub-project/datahub.git
synced 2025-07-26 11:00:06 +00:00
144 lines
185 KiB
HTML
144 lines
185 KiB
HTML
|
<!doctype html>
|
|||
|
|
<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-metadata-models-custom/README" data-has-hydrated="false">
|
|||
|
|
<head>
|
|||
|
|
<meta charset="UTF-8">
|
|||
|
|
<meta name="generator" content="Docusaurus v2.4.3">
|
|||
|
|
<title data-rh="true">A Custom Metadata Model | DataHub</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://docs.datahub.com/docs/metadata-models-custom"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="A Custom Metadata Model | DataHub"><meta data-rh="true" name="description" content="This module hosts a gradle project where you can store your custom metadata model. It contains an example extension for you to follow."><meta data-rh="true" property="og:description" content="This module hosts a gradle project where you can store your custom metadata model. It contains an example extension for you to follow."><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://docs.datahub.com/docs/metadata-models-custom"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/metadata-models-custom" hreflang="en"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/metadata-models-custom" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://RK0UG797F3-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/learn/rss.xml" title="DataHub RSS Feed">
|
|||
|
|
<link rel="alternate" type="application/atom+xml" href="/learn/atom.xml" title="DataHub Atom Feed">
|
|||
|
|
|
|||
|
|
<link rel="preconnect" href="https://www.google-analytics.com">
|
|||
|
|
<link rel="preconnect" href="https://www.googletagmanager.com">
|
|||
|
|
<script async src="https://www.googletagmanager.com/gtag/js?id=G-PKGVLETT4C"></script>
|
|||
|
|
<script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-PKGVLETT4C",{})</script>
|
|||
|
|
<link rel="preconnect" href="https://www.googletagmanager.com">
|
|||
|
|
<script>window.dataLayer=window.dataLayer||[]</script>
|
|||
|
|
<script>!function(e,t,a,n,g){e[n]=e[n]||[],e[n].push({"gtm.start":(new Date).getTime(),event:"gtm.js"});var m=t.getElementsByTagName(a)[0],r=t.createElement(a);r.async=!0,r.src="https://www.googletagmanager.com/gtm.js?id=GTM-5M8T9HNN",m.parentNode.insertBefore(r,m)}(window,document,"script","dataLayer")</script>
|
|||
|
|
|
|||
|
|
|
|||
|
|
<link rel="search" type="application/opensearchdescription+xml" title="DataHub" href="/opensearch.xml">
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
<meta httpequiv="Content-Security-Policy" content="frame-ancestors 'self' https://*.acryl.io https://acryldata.io http://localhost:*">
|
|||
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap">
|
|||
|
|
<script src="https://tools.luckyorange.com/core/lo.js?site-id=28ea8a38" async defer="defer"></script>
|
|||
|
|
<script src="/scripts/rb2b.js" async defer="defer"></script>
|
|||
|
|
<script src="https://app.revenuehero.io/scheduler.min.js"></script>
|
|||
|
|
<script src="https://tag.clearbitscripts.com/v1/pk_2e321cabe30432a5c44c0424781aa35f/tags.js" referrerpolicy="strict-origin-when-cross-origin"></script>
|
|||
|
|
<script src="/scripts/reo.js"></script>
|
|||
|
|
<script id="runllm-widget-script" type="module" src="https://widget.runllm.com" crossorigin="true" runllm-name="DataHub" runllm-assistant-id="81" runllm-position="BOTTOM_RIGHT" runllm-keyboard-shortcut="Mod+j" runllm-preset="docusaurus" runllm-theme-color="#1890FF" runllm-brand-logo="https://docs.datahub.com/img/datahub-logo-color-mark.svg" runllm-community-url="https://datahub.com/slack" runllm-community-type="slack" runllm-disable-ask-a-person="true" async></script><link rel="stylesheet" href="/assets/css/styles.d8fe2eb8.css">
|
|||
|
|
<link rel="preload" href="/assets/js/runtime~main.ec69bdbc.js" as="script">
|
|||
|
|
<link rel="preload" href="/assets/js/main.89423a85.js" as="script">
|
|||
|
|
</head>
|
|||
|
|
<body class="navigation-with-keyboard">
|
|||
|
|
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-5M8T9HNN" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
|||
|
|
|
|||
|
|
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
|
|||
|
|
<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:transparent;color:#ffffff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"><div class="shimmer-banner"><p>DataHub Secures $35 Million Series B</p><a href="https://datahub.com/news/series-b-announcement/" target="_blank" class="button"><div>Read the announcement<span> →</span></div></a></div></div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a href="https://datahub.com" target="_blank" rel="noopener noreferrer" class="navbar__brand"><div class="navbar__logo"><img src="/img/datahub-logo-color-light-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/datahub-logo-color-dark-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link versionNavItem_cbn8">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/metadata-models-custom">Next</a></li><li><a class="dropdown__link" href="/docs/1.1.0/metadata-models-custom">1.1.0</a></li><li><hr class="dropdown-separator" style="margin: 0.4rem;"></li><li><div class="dropdown__link"><b>Archived versions</b></div></li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/features">1.0.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/0.15.0/features">0.15.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-8jkm4uler-acryldata.vercel.app/docs/0.14.1/features">0.14.1
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-eue2qafvn-acryldata.vercel.app/docs/features">0.14.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-psat3nzgi-acryldata.vercel.app/docs/features">0.13.1
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-lzxh86531-acryldata.vercel.app/docs/features">0.13.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-2uuxmgza2-acryldata.vercel.app/docs/features">0.12.1
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-irpoe2osc-acryldata.vercel.app/docs/features">0.11.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-1gv2yzn9d-acryldata.vercel.app/docs/features">0.10.5
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li></ul></div></div><div class="navbar__items navbar__items--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/docs">Docs</a><a class="navbar__item navbar__link" href="/integrations">Integrations</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/weekly-demo" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Weekly Demo"></div><div class="title_c7DP">Weekly Demo</div></a></div><div><a href="https://datahub.com/use-cases" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-forum.png" alt="Use Cases"></div><div class="title_c7DP">Use Cases</div></a></div><div><a href="httpps://datahub.com/adoption-stories" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Adoption Stories"></div><div class="title_c7DP">Adoption Stories</div></a></div><div><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Blog"></div><div class="title_c7DP">Blog</div></a></div><div><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Youtube"></div><div class="title_c7DP">Youtube</div></a></div></div></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/slack/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Join Slack"></div><div class="title_c7DP">Join Slack</div></a></div><div><a href="https://datahub.com/events" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Events"></div><div class="title_c7DP">Events</div></a></div><div><a href="https://datahub.com/champions/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Champions"></div><div class="title_c7DP">Champions</div></a></div><div><a href="https://datahub.com/share-your-journey/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Share Your Journey"></div><div class="title_c7DP">Share Your Journey</div></a></div></div></ul></div><a href="https://datahub.com/products/why-datahub-cloud/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
|||
|
|
<style>
|
|||
|
|
.cloud-cta {
|
|||
|
|
color: var(--ifm-menu-color-active);
|
|||
|
|
font-weight: 600;
|
|||
|
|
background: linear-gradient(40deg, var(--ifm-menu-color-active), var(--ifm-menu-color-active));
|
|||
|
|
background-size: 200% 100%;
|
|||
|
|
-webkit-background-clip: text;
|
|||
|
|
background-clip: text;
|
|||
|
|
transition: background-image 0.3s ease;
|
|||
|
|
}
|
|||
|
|
.cloud-cta:hover {
|
|||
|
|
color: transparent;
|
|||
|
|
background: linear-gradient(40deg, var(--ifm-menu-color-active), #ff1493);
|
|||
|
|
background-size: 200% 100%;
|
|||
|
|
-webkit-background-clip: text;
|
|||
|
|
background-clip: text;
|
|||
|
|
animation: gradientShift 3s ease infinite;
|
|||
|
|
}
|
|||
|
|
@keyframes gradientShift {
|
|||
|
|
0%, 100% { background-position: 0% 50%; }
|
|||
|
|
50% { background-position: 100% 50%; }
|
|||
|
|
}
|
|||
|
|
</style>
|
|||
|
|
<div class="cloud-cta">Get Cloud</div>
|
|||
|
|
</a><a href="https://datahub.com/slack?utm_source=docs&utm_medium=header&utm_campaign=docs_header" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
|||
|
|
<style>
|
|||
|
|
.slack-logo:hover {
|
|||
|
|
opacity: 0.8;
|
|||
|
|
}
|
|||
|
|
</style>
|
|||
|
|
<img class="slack-logo" src="https://upload.wikimedia.org/wikipedia/commons/d/d5/Slack_icon_2019.svg" , alt="slack" , height="20px" style="margin: 10px 0 0 0;">
|
|||
|
|
</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>A Custom Metadata Model</h1><p><a href="https://github.com/datahub-project/datahub/blob/master/metadata-models-custom/" target="_blank" rel="noopener noreferrer">This module</a> hosts a gradle project where you can store your custom metadata model. It contains an example extension for you to follow.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="caveats">Caveats<a href="#caveats" class="hash-link" aria-label="Direct link to Caveats" title="Direct link to Caveats"></a></h3><p>Currently, this project only supports aspects defined in PDL to existing or newly defined entities. You cannot add new aspects to the metadata model directly through yaml configuration yet.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="pre-requisites">Pre-Requisites<a href="#pre-requisites" class="hash-link" aria-label="Direct link to Pre-Requisites" title="Direct link to Pre-Requisites"></a></h2><p>Before proceeding further, make sure you understand the DataHub Metadata Model concepts defined <a href="/docs/metadata-modeling/metadata-model">here</a> and extending the model defined <a href="/docs/metadata-modeling/extending-the-metadata-model">here</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-your-new-aspects">Create your new aspect(s)<a href="#create-your-new-aspects" class="hash-link" aria-label="Direct link to Create your new aspect(s)" title="Direct link to Create your new aspect(s)"></a></h2><p>Follow the regular process in creating a new aspect by adding it to the <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models-custom/src/main/pegasus" target="_blank" rel="noopener noreferrer"><code>src/main/pegasus</code></a> folder. e.g. This repository has an Aspect called <code>customDataQualityRules</code> hosted in the <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models-custom/src/main/pegasus/com/mycompany/dq/DataQualityRules.pdl" target="_blank" rel="noopener noreferrer"><code>DataQualityRules.pdl</code></a> file that you can follow.
|
|||
|
|
Once you've gone through this exercise, feel free to delete the sample aspects that are stored in this module.</p><p><strong><em>Tip</em></strong>: PDL requires that the name of the file must match the name of the class that is defined in it and the package path must also match the directory path, so keep that in mind when you create your aspect pdl file.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="add-your-aspects-to-the-entity-registry">Add your aspect(s) to the entity registry<a href="#add-your-aspects-to-the-entity-registry" class="hash-link" aria-label="Direct link to Add your aspect(s) to the entity registry" title="Direct link to Add your aspect(s) to the entity registry"></a></h2><p>Add your new aspect(s) to the entity registry by editing the yaml file located under <code>registry/entity-registry.yaml</code>.
|
|||
|
|
<strong><em>Note</em></strong>: The registry file must be called <code>entity-registry.yaml</code> or <code>entity-registry.yml</code> for it to be recognized.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="understanding-the-entity-registry">Understanding the entity registry<a href="#understanding-the-entity-registry" class="hash-link" aria-label="Direct link to Understanding the entity registry" title="Direct link to Understanding the entity registry"></a></h2><p>Here is a sample entity-registry file</p><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token key atrule">id</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> mycompany</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">dq</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">model</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">entities</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> dataset</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">aspects</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> customDataQualityRules</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The entity registry has a few important fields to pay attention to:</p><ul><li>id: The name of your registry. This drives naming, artifact generation, so make sure you pick a unique name that will not conflict with other names you might create for other registries.</li><li>entities: A list of entities with aspects attached to them that you are creating additional aspects for as well as any new entities you wish to define. In this example, we are adding the aspect <code>customDataQualityRules</code> to the <code>dataset</code> entity.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="build-your-new-model">Build your new model<a href="#build-your-new-model" class="hash-link" aria-label="Direct link to Build your new model" title="Direct link to Build your new model"></a></h2><p>Change your directory to the metadata-models-custom folder and then run this command</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="co
|
|||
|
|
(generated from <code>../gradlew modelArtifact</code> under <code>build/dist</code>) and place the unzipped contents in the volumes mount for the GMS pod on the remote server.
|
|||
|
|
First you will need to push the files into a configmap using kubectl:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">kubectl create configmap custom-model --from-file=<<path-to-file>> -n <<namespace>></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then you need to set the volumes for GMS (refer to how jmx exporter configmap is added here:
|
|||
|
|
<a href="https://github.com/acryldata/datahub-helm/blob/master/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml#L40" target="_blank" rel="noopener noreferrer">https://github.com/acryldata/datahub-helm/blob/master/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml#L40</a>)
|
|||
|
|
This tells GMS that we will be pulling this configmap in. You can do this by setting <code>datahub-gms.extraVolumes</code> in <code>values.yaml</code>
|
|||
|
|
which gets appended to the deployment without having to change the helm chart.</p><p>Finally you need to mount the volume into the container’s local directory by setting volumeMounts.
|
|||
|
|
Refer to how the kafka certs are mounted onto a local path here:
|
|||
|
|
<a href="https://github.com/acryldata/datahub-helm/blob/master/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml#L182" target="_blank" rel="noopener noreferrer">https://github.com/acryldata/datahub-helm/blob/master/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml#L182</a>
|
|||
|
|
You can do this by setting the datahub-gms.extraVolumeMounts in <code>values.yaml</code></p><p>at the end your values.yaml should have something like:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub-gms:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> extraVolumes:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> - name: custom-model</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> configMap:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> name: custom-model ## should match configmap name above</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> extraVolumeMounts:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> - name: custom-model-dir</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> mountPath: /etc/plugins/models/<registry-name>/<registry-version></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The mountPath can be configured using <code>ENTITY_REGISTRY_PLUGIN_PATH</code> and defaults to <code>/etc/datahub/plugins/models</code>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="check-if-your-model-got-loaded-successfully">Check if your model got loaded successfully<a href="#check-if-your-model-got-loaded-successfully" class="hash-link" aria-label="Direct link to Check if your model got loaded successfully" title="Direct link to Check if your model got loaded successfully"></a></h3><p>Assuming that you are running DataHub on localhost, you can curl the config endpoint to see the model load status.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">curl -s http://localhost:8080/config | jq .</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span
|
|||
|
|
The <code>scripts/insert_one.sh</code> script takes the <code>scripts/data/dq_rule.json</code> file and attaches it to the <code>dataset_urn</code> entity using the <code>datahub</code> cli.</p><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">cd scripts</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">./insert_one.sh</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>results in</p><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">Update succeeded with status 200</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The <code>scripts/insert_custom_aspect.py</code> script shows you how to accomplish the same using the Python SDK. Note that we are just using a raw dictionary here to represent the <code>dq_rule</code> aspect and not a strongly-typed class.</p><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">cd scripts</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">python3 insert_custom_aspect.py</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>results in</p><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">Successfully wrote to DataHub</span><br></span></co
|
|||
|
|
just the data model is not enough and additional custom code might be required. For a few of these use cases a plugin framework was developed
|
|||
|
|
to control how instances of custom aspects can be validated, mutated, and generate side effects (additional aspects).</p><p>It should be noted that validation, mutation, and generation of the <em>core</em> DataHub aspects can lead to system corruption and should be used
|
|||
|
|
by advanced users only.</p><p>The <code>/config</code> endpoint documented above has been extended to return information on the instances of the various plugins as well as the classes
|
|||
|
|
that were loaded for debugging purposes.</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"mycompany-dq-model"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"0.0.0-dev"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"plugins"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"validatorCount"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"mutationHookCount"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"mcpSideEffectCount"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"mclSideEffectCount"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"validatorClasses"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"com.linkedin.metadata.aspect.plugins.validation.CustomDataQualityRulesValidator"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">
|
|||
|
|
the context of typical read/write operations within DataHub. Each component is discussed in further detail in the
|
|||
|
|
sections below.</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/metadata-models-custom/custom-plugin-overview.svg" class="img_ev3q"></p><p>In the diagram above, the circles represent Aspects (custom aspects or standard). As the Aspects progress they can be mutated/changed,
|
|||
|
|
rejected, or additional aspects can be generated by side effects.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="custom-validators">Custom Validators<a href="#custom-validators" class="hash-link" aria-label="Direct link to Custom Validators" title="Direct link to Custom Validators"></a></h4><p>Custom aspects might require that instances of those aspects adhere to specific conditions or rules. These conditions could vary wildly depending on the use case however they could be as simple
|
|||
|
|
as a null or range check for one or more fields within the custom aspect. Additionally, a lookup can be done on other aspects in order to validate the current aspect using the <code>AspectRetriever</code>.</p><p>There are two integration points for validation. The first integration point is <code>on request</code> via the <code>validateProposedAspect</code> method where the aspect is validated independent of the previous value. This validation is performed
|
|||
|
|
outside of a database transaction and can perform more intensive checks without introducing added latency within a transaction. Note that added latency from the validation check is still introduced into the request itself.</p><p>The second integration point for validation occurs within the database transaction using the <code>validatePreCommitAspect</code> and has access to the new aspect as well as the old aspect. See the included
|
|||
|
|
example in <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/validation/CustomDataQualityRulesValidator.java" target="_blank" rel="noopener noreferrer"><code>CustomDataQualityRulesValidator.java</code></a>.</p><p>Shown below is the interface to be implemented for a custom validator.</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">public</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">class</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">CustomDataQualityRulesValidator</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">extends</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">AspectPayloadValidator</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Override</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token keyword" style="font-style:italic">protected</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">Stream</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token generics class-name" style="color:rgb(255, 203, 107)">AspectValidationException</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">></span><span class="token plain"> </span><span class="token function" style="color:rgb(130, 170, 255)">validateProposedAspects</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Nonnull</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">Collection</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token generics operator" style="color:rgb(137, 221, 255)">?</span><span class="token generics"> </span><span class="token generics keyword" style="font-style:italic">extends</span><span class="token generics"> </span><span class="token generics class-name" style="color:rgb(255, 203, 107)">BatchItem</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">></span><span class="token plain"> mcpItems</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Nonnull</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">AspectRetriever</span><span class="token plain"> aspectRetriever</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="displ
|
|||
|
|
the validator to run on upsert operations for any entity with the custom aspect <code>customDataQualityRules</code>. Alternatively separate
|
|||
|
|
validators could be written within the context of specific entities, in this case simply specify the entity name instead of <code>*</code>.</p><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token key atrule">plugins</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">aspectPayloadValidators</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">className</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"com.linkedin.metadata.aspect.plugins.validation.CustomDataQualityRulesValidator"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">enabled</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token boolean important" style="color:rgb(255, 88, 116)">true</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">supportedOperations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> UPSERT</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">supportedEntityAspectNames</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">entityName</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"*"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">aspectName</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> customDataQualityRules</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="custom-mutator">Custom Mutator<a href="#custom-mutator" class="hash-link" aria-label="Direct link to Custom Mutator" title="Direct link to Custom Mutato
|
|||
|
|
being written to persistent storage. The second mutation hook is a <code>read</code> hook which can modify the data when
|
|||
|
|
read from persistent storage.</p><p>Write Mutation:</p><p>In this example, we want to make sure that the field type is always lowercase regardless of the string being provided
|
|||
|
|
by ingestion. The full example can be found in <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMutator.java" target="_blank" rel="noopener noreferrer"><code>CustomDataQualityMutator.java</code></a>.</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">public</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">class</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">CustomDataQualityRulesMutator</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">extends</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">MutationHook</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Override</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token keyword" style="font-style:italic">protected</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">Stream</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token generics class-name" style="color:rgb(255, 203, 107)">Pair</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token generics class-name" style="color:rgb(255, 203, 107)">ChangeMCP</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token generics"> </span><span class="token generics class-name" style="color:rgb(255, 203, 107)">Boolean</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">></span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">></span><span class="token plain"> </span><span class="token function" style="color:rgb(130, 170, 255)">writeMutation</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Nonnull</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">Collection</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token generics class-name" style="color:rgb(255, 203, 107)">ChangeMCP</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">></span><span class="token plain"> changeMCPS</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Nonnull</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">AspectRetriever</span><span class="token plain"> aspectRetriever</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token keyword" style="font-style:italic">retur
|
|||
|
|
deleted structured properties from being returned on entities.</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">public</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">class</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">StructuredPropertiesSoftDelete</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">extends</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">MutationHook</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Override</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token keyword" style="font-style:italic">protected</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">Stream</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token generics class-name" style="color:rgb(255, 203, 107)">Pair</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token generics class-name" style="color:rgb(255, 203, 107)">ReadItem</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token generics"> </span><span class="token generics class-name" style="color:rgb(255, 203, 107)">Boolean</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">></span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">></span><span class="token plain"> </span><span class="token function" style="color:rgb(130, 170, 255)">readMutation</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Nonnull</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">Collection</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token generics class-name" style="color:rgb(255, 203, 107)">ReadItem</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">></span><span class="token plain"> items</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Nonnull</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">AspectRetriever</span><span class="token plain"> aspectRetriever</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">Map</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token generics class-name" style="color:rgb(255, 203, 107)">Urn</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">,</s
|
|||
|
|
or modified we'll record the actor, event type, and timestamp in a timeseries aspect index.</p><p>Notes:</p><ul><li>MCLs are only persisted to the search indices which allows for adding to the search documents only.</li><li>Dependency on the <code>metadata-io</code> module since it deals with lower level storage primitives.</li></ul><p>The full example can be found in <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCLSideEffect.java" target="_blank" rel="noopener noreferrer"><code>CustomDataQualityRulesMCLSideEffect.java</code></a>.</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">public</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">class</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">CustomDataQualityRulesMCLSideEffect</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">extends</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">MCLSideEffect</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Override</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token keyword" style="font-style:italic">protected</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">Stream</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token generics class-name" style="color:rgb(255, 203, 107)">MCLItem</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">></span><span class="token plain"> </span><span class="token function" style="color:rgb(130, 170, 255)">applyMCLSideEffect</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Nonnull</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">Collection</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token generics class-name" style="color:rgb(255, 203, 107)">MCLItem</span><span class="token generics punctuation" style="color:rgb(199, 146, 234)">></span><span class="token plain"> mclItems</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Nonnull</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">AspectRetriever</span><span class="token plain"> aspectRetriever</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token keyword" style="font-style:italic">return</span><span class="token plain"> mclItems</span><span class="token punctuation" style="color:rgb(199, 146,
|
|||
|
|
not required, it is possible to use Spring to both inject autoconfiguration and the plugins themselves. An example Spring-enabled
|
|||
|
|
validator has been included in the package <code>com.linkedin.metadata.aspect.plugins.spring.validation</code>. The plugin
|
|||
|
|
class loader and Spring context is isolated so conflicts between DataHub and custom classes are avoided.</p><p>The configuration of a Spring enabled plugin looks like the following, note the addition of <code>spring.enabled: true</code> below.
|
|||
|
|
A list of packages to scan for Spring configuration and components should also be provided which should include
|
|||
|
|
your custom classes with Spring annotations per the <code>packageScan</code> below.</p><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token key atrule">plugins</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">aspectPayloadValidators</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">className</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"com.linkedin.metadata.aspect.plugins.spring.validation.CustomDataQualityRulesValidator"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">packageScan</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> com.linkedin.metadata.aspect.plugins.spring.validation</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">enabled</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token boolean important" style="color:rgb(255, 88, 116)">true</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">supportedOperations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> UPSERT</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">supportedEntityAspectNames</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">entityName</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"dataset"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">aspectName</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> customDataQualityRules</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">spring</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </sp
|
|||
|
|
injection of a bean into a function which is called by Spring after constructing the custom validator plugin.</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Configuration</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token keyword" style="font-style:italic">public</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">class</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">CustomDataQualityRulesConfig</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Bean</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token string" style="color:rgb(195, 232, 141)">"myCustomMessage"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token keyword" style="font-style:italic">public</span><span class="token plain"> </span><span class="token class-name" style="color:rgb(255, 203, 107)">String</span><span class="token plain"> </span><span class="token function" style="color:rgb(130, 170, 255)">myCustomMessage</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token keyword" style="font-style:italic">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"Spring injection works!"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token annotation punctuation" style="color:rgb(199, 146, 234)">@Component</span><span class="token plain"></span><br></span><span class="
|
|||
|
|
<script src="/assets/js/runtime~main.ec69bdbc.js"></script>
|
|||
|
|
<script src="/assets/js/main.89423a85.js"></script>
|
|||
|
|
</body>
|
|||
|
|
</html>
|