mirror of
https://github.com/datahub-project/datahub.git
synced 2025-07-12 19:49:36 +00:00
140 lines
205 KiB
HTML
140 lines
205 KiB
HTML
<!doctype html>
|
||
<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-docs/cli" data-has-hydrated="false">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="generator" content="Docusaurus v2.4.3">
|
||
<title data-rh="true">DataHub CLI | DataHub</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://docs.datahub.com/docs/cli"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="DataHub CLI | DataHub"><meta data-rh="true" name="description" content="DataHub comes with a friendly cli called datahub that allows you to perform a lot of common operations using just the command line. DataHub maintains the pypi package for datahub."><meta data-rh="true" property="og:description" content="DataHub comes with a friendly cli called datahub that allows you to perform a lot of common operations using just the command line. DataHub maintains the pypi package for datahub."><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://docs.datahub.com/docs/cli"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/cli" hreflang="en"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/cli" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://RK0UG797F3-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/learn/rss.xml" title="DataHub RSS Feed">
|
||
<link rel="alternate" type="application/atom+xml" href="/learn/atom.xml" title="DataHub Atom Feed">
|
||
|
||
<link rel="preconnect" href="https://www.google-analytics.com">
|
||
<link rel="preconnect" href="https://www.googletagmanager.com">
|
||
<script async src="https://www.googletagmanager.com/gtag/js?id=G-PKGVLETT4C"></script>
|
||
<script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-PKGVLETT4C",{})</script>
|
||
<link rel="preconnect" href="https://www.googletagmanager.com">
|
||
<script>window.dataLayer=window.dataLayer||[]</script>
|
||
<script>!function(e,t,a,n,g){e[n]=e[n]||[],e[n].push({"gtm.start":(new Date).getTime(),event:"gtm.js"});var m=t.getElementsByTagName(a)[0],r=t.createElement(a);r.async=!0,r.src="https://www.googletagmanager.com/gtm.js?id=GTM-5M8T9HNN",m.parentNode.insertBefore(r,m)}(window,document,"script","dataLayer")</script>
|
||
|
||
|
||
<link rel="search" type="application/opensearchdescription+xml" title="DataHub" href="/opensearch.xml">
|
||
|
||
|
||
|
||
|
||
<meta httpequiv="Content-Security-Policy" content="frame-ancestors 'self' https://*.acryl.io https://acryldata.io http://localhost:*">
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap">
|
||
<script src="https://tools.luckyorange.com/core/lo.js?site-id=28ea8a38" async defer="defer"></script>
|
||
<script src="/scripts/rb2b.js" async defer="defer"></script>
|
||
<script src="https://app.revenuehero.io/scheduler.min.js"></script>
|
||
<script src="https://tag.clearbitscripts.com/v1/pk_2e321cabe30432a5c44c0424781aa35f/tags.js" referrerpolicy="strict-origin-when-cross-origin"></script>
|
||
<script src="/scripts/reo.js"></script>
|
||
<script id="runllm-widget-script" type="module" src="https://widget.runllm.com" crossorigin="true" runllm-name="DataHub" runllm-assistant-id="81" runllm-position="BOTTOM_RIGHT" runllm-keyboard-shortcut="Mod+j" runllm-preset="docusaurus" runllm-theme-color="#1890FF" runllm-brand-logo="https://docs.datahub.com/img/datahub-logo-color-mark.svg" runllm-community-url="https://datahub.com/slack" runllm-community-type="slack" runllm-disable-ask-a-person="true" async></script><link rel="stylesheet" href="/assets/css/styles.d8fe2eb8.css">
|
||
<link rel="preload" href="/assets/js/runtime~main.ec69bdbc.js" as="script">
|
||
<link rel="preload" href="/assets/js/main.89423a85.js" as="script">
|
||
</head>
|
||
<body class="navigation-with-keyboard">
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-5M8T9HNN" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
|
||
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
|
||
<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:transparent;color:#ffffff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"><div class="shimmer-banner"><p>DataHub Secures $35 Million Series B</p><a href="https://datahub.com/news/series-b-announcement/" target="_blank" class="button"><div>Read the announcement<span> →</span></div></a></div></div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a href="https://datahub.com" target="_blank" rel="noopener noreferrer" class="navbar__brand"><div class="navbar__logo"><img src="/img/datahub-logo-color-light-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/datahub-logo-color-dark-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link versionNavItem_cbn8">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/cli">Next</a></li><li><a class="dropdown__link" href="/docs/1.1.0/cli">1.1.0</a></li><li><hr class="dropdown-separator" style="margin: 0.4rem;"></li><li><div class="dropdown__link"><b>Archived versions</b></div></li><li>
|
||
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/features">1.0.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/0.15.0/features">0.15.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-8jkm4uler-acryldata.vercel.app/docs/0.14.1/features">0.14.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-eue2qafvn-acryldata.vercel.app/docs/features">0.14.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-psat3nzgi-acryldata.vercel.app/docs/features">0.13.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-lzxh86531-acryldata.vercel.app/docs/features">0.13.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-2uuxmgza2-acryldata.vercel.app/docs/features">0.12.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-irpoe2osc-acryldata.vercel.app/docs/features">0.11.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-1gv2yzn9d-acryldata.vercel.app/docs/features">0.10.5
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li></ul></div></div><div class="navbar__items navbar__items--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/docs">Docs</a><a class="navbar__item navbar__link" href="/integrations">Integrations</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/weekly-demo" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Weekly Demo"></div><div class="title_c7DP">Weekly Demo</div></a></div><div><a href="https://datahub.com/use-cases" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-forum.png" alt="Use Cases"></div><div class="title_c7DP">Use Cases</div></a></div><div><a href="httpps://datahub.com/adoption-stories" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Adoption Stories"></div><div class="title_c7DP">Adoption Stories</div></a></div><div><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Blog"></div><div class="title_c7DP">Blog</div></a></div><div><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Youtube"></div><div class="title_c7DP">Youtube</div></a></div></div></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/slack/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Join Slack"></div><div class="title_c7DP">Join Slack</div></a></div><div><a href="https://datahub.com/events" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Events"></div><div class="title_c7DP">Events</div></a></div><div><a href="https://datahub.com/champions/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Champions"></div><div class="title_c7DP">Champions</div></a></div><div><a href="https://datahub.com/share-your-journey/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Share Your Journey"></div><div class="title_c7DP">Share Your Journey</div></a></div></div></ul></div><a href="https://datahub.com/products/why-datahub-cloud/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
||
<style>
|
||
.cloud-cta {
|
||
color: var(--ifm-menu-color-active);
|
||
font-weight: 600;
|
||
background: linear-gradient(40deg, var(--ifm-menu-color-active), var(--ifm-menu-color-active));
|
||
background-size: 200% 100%;
|
||
-webkit-background-clip: text;
|
||
background-clip: text;
|
||
transition: background-image 0.3s ease;
|
||
}
|
||
.cloud-cta:hover {
|
||
color: transparent;
|
||
background: linear-gradient(40deg, var(--ifm-menu-color-active), #ff1493);
|
||
background-size: 200% 100%;
|
||
-webkit-background-clip: text;
|
||
background-clip: text;
|
||
animation: gradientShift 3s ease infinite;
|
||
}
|
||
@keyframes gradientShift {
|
||
0%, 100% { background-position: 0% 50%; }
|
||
50% { background-position: 100% 50%; }
|
||
}
|
||
</style>
|
||
<div class="cloud-cta">Get Cloud</div>
|
||
</a><a href="https://datahub.com/slack?utm_source=docs&utm_medium=header&utm_campaign=docs_header" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
||
<style>
|
||
.slack-logo:hover {
|
||
opacity: 0.8;
|
||
}
|
||
</style>
|
||
<img class="slack-logo" src="https://upload.wikimedia.org/wikipedia/commons/d/d5/Slack_icon_2019.svg" , alt="slack" , height="20px" style="margin: 10px 0 0 0;">
|
||
</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Getting Started</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/features">What Is DataHub?</a><button aria-label="Toggle the collapsible sidebar category 'What Is DataHub?'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/features">Features</a><button aria-label="Toggle the collapsible sidebar category 'Features'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>DataHub Cloud</div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/managed-datahub-overview">DataHub Cloud Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/welcome-acryl">Getting Started with DataHub Cloud</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authentication/guides/sso/initialize-oidc">Configure Single Sign-On</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/remote-executor/about">Remote Executor</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/datahub-api/entity-events-api">DataHub API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/slack/saas-slack-app">Slack</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge">Operator Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item saasOnly"><a class="menu__link" href="/docs/managed-datahub/approval-workflows">Change Proposals & Approval Workflows</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/chrome-extension">Cloud Chrome Extension</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item saasOnly"><a class="menu__link" href="/docs/managed-datahub/subscription-and-notification">Subscriptions & Notifications</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/release-notes/v_0_3_12">DataHub Cloud Release History</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Integrations</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-ingestion">Overview</a><button aria-label="Toggle the collapsible sidebar category 'Overview'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/cli-ingestion">Quickstart Guides</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-ingestion/source_overview">Sources</a><button aria-label="Toggle the collapsible sidebar category 'Sources'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/schedule_docs/intro">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>API & SDKs</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-modeling/metadata-model">DataHub's Open Metadata Standard</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/what-is-datahub/datahub-concepts">Concepts</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-standards">Metadata Standards</a><button aria-label="Toggle the collapsible sidebar category 'Metadata Standards'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/api/datahub-apis">APIs and SDKs Overview</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/api/graphql/overview">API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/as-a-library">Python SDK</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/metadata-integration/java/as-a-library">Java SDK</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible menu__list-item-collapsible--active"><a class="menu__link menu__link--sublist menu__link--active" aria-current="page" aria-expanded="true" href="/docs/cli">DataHub CLI</a><button aria-label="Toggle the collapsible sidebar category 'DataHub CLI'" type="button" class="clean-btn menu__caret"></button></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/cli-commands/dataset">Dataset Command</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/datahub_lite">Lite (Experimental)</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/act-on-metadata">DataHub Actions</a><button aria-label="Toggle the collapsible sidebar category 'DataHub Actions'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/api/tutorials/datasets">Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Admin</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authentication">Authentication</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authorization">Authorization</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/how/delete-metadata">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Deployment</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/deployment-guides">Deployment Guides</a><button aria-label="Toggle the collapsible sidebar category 'Deployment Guides'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/deploy/confluent-cloud">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Developers</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/architecture/architecture">Architecture</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/developers">Developing on DataHub</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/advanced/mcp-mcl">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Community</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/community">Community</a><button aria-label="Toggle the collapsible sidebar category 'Community'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/releases">Release History</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">DataHub CLI</span><meta itemprop="position" content="1"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>DataHub CLI</h1><p>DataHub comes with a friendly cli called <code>datahub</code> that allows you to perform a lot of common operations using just the command line. <a href="https://datahub.com" target="_blank" rel="noopener noreferrer">DataHub</a> maintains the <a href="https://pypi.org/project/acryl-datahub/" target="_blank" rel="noopener noreferrer">pypi package</a> for <code>datahub</code>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="installation">Installation<a href="#installation" class="hash-link" aria-label="Direct link to Installation" title="Direct link to Installation"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="using-pip">Using pip<a href="#using-pip" class="hash-link" aria-label="Direct link to Using pip" title="Direct link to Using pip"></a></h3><p>We recommend Python virtual environments (venv-s) to namespace pip modules. Here's an example setup:</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">python3 -m venv venv </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># create the environment</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">source</span><span class="token plain"> venv/bin/activate </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># activate the environment</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><strong><em>NOTE:</em></strong> If you install <code>datahub</code> in a virtual environment, that same virtual environment must be re-activated each time a shell window or session is created.</p><p>Once inside the virtual environment, install <code>datahub</code> using the following commands</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Requires Python 3.9+</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">python3 -m pip </span><span class="token function" style="color:rgb(130, 170, 255)">install</span><span class="token plain"> --upgrade pip wheel setuptools</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">python3 -m pip </span><span class="token function" style="color:rgb(130, 170, 255)">install</span><span class="token plain"> --upgrade acryl-datahub</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># validate that the install was successful</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub version</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># If you see "command not found", try running this instead: python3 -m datahub version</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub init</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># authenticate your datahub CLI with your datahub instance</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>If you run into an error, try checking the <a href="/docs/metadata-ingestion/developing#common-setup-issues"><em>common setup issues</em></a>.</p><p>Other installation options such as installation from source and running the cli inside a container are available further below in the guide <a href="#alternate-installation-options">here</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="starter-commands">Starter Commands<a href="#starter-commands" class="hash-link" aria-label="Direct link to Starter Commands" title="Direct link to Starter Commands"></a></h2><p>The <code>datahub</code> cli allows you to do many things, such as quick-starting a DataHub docker instance locally, ingesting metadata from your sources into a DataHub server or a DataHub lite instance, as well as retrieving, modifying and exploring metadata.
|
||
Like most command line tools, <code>--help</code> is your best friend. Use it to discover the capabilities of the cli and the different commands and sub-commands that are supported.</p><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub --help</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Usage: datahub [OPTIONS] COMMAND [ARGS]...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Options:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --debug / --no-debug Enable debug logging.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --log-file FILE Enable debug logging.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --debug-vars / --no-debug-vars Show variable values in stack traces. Implies --debug. While we try to avoid</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> printing sensitive information like passwords, this may still happen.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --version Show the version and exit.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> -dl, --detect-memory-leaks Run memory leak detection.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --help Show this message and exit.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Commands:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> actions <disabled due to missing dependencies></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> assertions A group of commands to interact with the Assertion entity in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> check Helper commands for checking various aspects of DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> container A group of commands to interact with containers in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> datacontract A group of commands to interact with the DataContract entity in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> dataproduct A group of commands to interact with the DataProduct entity in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> dataset A group of commands to interact with the Dataset entity in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> delete Delete metadata from DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> docker Helper commands for setting up and interacting with a local DataHub instance using Docker.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> exists A group of commands to check existence of entities in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> forms A group of commands to interact with forms in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> get A group of commands to get metadata from DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> group A group of commands to interact with the Group entity in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ingest Ingest metadata into DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> init Configure which datahub instance to connect to</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> lite A group of commands to work with a DataHub Lite instance</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> migrate Helper commands for migrating metadata within DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> properties A group of commands to interact with structured properties in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> put A group of commands to put metadata in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> state Managed state stored in DataHub by stateful ingestion.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> telemetry Toggle telemetry.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> timeline Get timeline for an entity based on certain categories</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> user A group of commands to interact with the User entity in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> version Print version number and exit.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The following top-level commands listed below are here mainly to give the reader a high-level picture of what are the kinds of things you can accomplish with the cli.
|
||
We've ordered them roughly in the order we expect you to interact with these commands as you get deeper into the <code>datahub</code>-verse.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="docker">docker<a href="#docker" class="hash-link" aria-label="Direct link to docker" title="Direct link to docker"></a></h3><p>The <code>docker</code> command allows you to start up a local DataHub instance using <code>datahub docker quickstart</code>. You can also check if the docker cluster is healthy using <code>datahub docker check</code>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ingest">ingest<a href="#ingest" class="hash-link" aria-label="Direct link to ingest" title="Direct link to ingest"></a></h3><p>The <code>ingest</code> command allows you to ingest metadata from your sources using ingestion configuration files, which we call recipes.
|
||
Source specific crawlers are provided by plugins and might sometimes need additional extras to be installed. See <a href="#installing-plugins">installing plugins</a> for more information.
|
||
<a href="/docs/how/delete-metadata">Removing Metadata from DataHub</a> contains detailed instructions about how you can use the ingest command to perform operations like rolling-back previously ingested metadata through the <code>rollback</code> sub-command and listing all runs that happened through <code>list-runs</code> sub-command.</p><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">Usage: datahub [datahub-options] ingest [command-options]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Command Options:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> -c / --config Config file in .toml or .yaml format</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> -n / --dry-run Perform a dry run of the ingestion, essentially skipping writing to sink</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --preview Perform limited ingestion from the source to the sink to get a quick preview</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --preview-workunits The number of workunits to produce for preview</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --strict-warnings If enabled, ingestion runs with warnings will yield a non-zero error code</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --test-source-connection When set, ingestion will only test the source connection details from the recipe</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --no-progress If enabled, mute intermediate progress ingestion reports</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ingest---dry-run">ingest --dry-run<a href="#ingest---dry-run" class="hash-link" aria-label="Direct link to ingest --dry-run" title="Direct link to ingest --dry-run"></a></h4><p>The <code>--dry-run</code> option of the <code>ingest</code> command performs all of the ingestion steps, except writing to the sink. This is useful to validate that the
|
||
ingestion recipe is producing the desired metadata events before ingesting them into datahub.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Dry run</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml --dry-run</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Short-form</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml -n</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ingest-list-source-runs">ingest list-source-runs<a href="#ingest-list-source-runs" class="hash-link" aria-label="Direct link to ingest list-source-runs" title="Direct link to ingest list-source-runs"></a></h4><p>The <code>list-source-runs</code> option of the <code>ingest</code> command lists the previous runs, displaying their run ID, source name,
|
||
start time, status, and source URN. This command allows you to filter results using the --urn option for URN-based
|
||
filtering or the --source option to filter by source name (partial or complete matches are supported).</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># List all ingestion runs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest list-source-runs</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Filter runs by a source name containing "demo"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest list-source-runs --source </span><span class="token string" style="color:rgb(195, 232, 141)">"demo"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ingest---preview">ingest --preview<a href="#ingest---preview" class="hash-link" aria-label="Direct link to ingest --preview" title="Direct link to ingest --preview"></a></h4><p>The <code>--preview</code> option of the <code>ingest</code> command performs all of the ingestion steps, but limits the processing to only the first 10 workunits produced by the source.
|
||
This option helps with quick end-to-end smoke testing of the ingestion recipe.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Preview</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml --preview</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Preview with dry-run</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml -n --preview</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>By default <code>--preview</code> creates 10 workunits. But if you wish to try producing more workunits you can use another option <code>--preview-workunits</code></p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Preview 20 workunits without sending anything to sink</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml -n --preview --preview-workunits</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">20</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ingest---no-default-report">ingest --no-default-report<a href="#ingest---no-default-report" class="hash-link" aria-label="Direct link to ingest --no-default-report" title="Direct link to ingest --no-default-report"></a></h4><p>By default, the cli sends an ingestion report to DataHub, which allows you to see the result of all cli-based ingestion in the UI. This can be turned off with the <code>--no-default-report</code> flag.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Running ingestion with reporting to DataHub turned off</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml --no-default-report</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The reports include the recipe that was used for ingestion. This can be turned off by adding an additional section to the ingestion recipe.</p><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token key atrule">source</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># source configs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">sink</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># sink configs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Add configuration for the datahub reporter</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">reporting</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">type</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> datahub</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">config</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">report_recipe</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token boolean important" style="color:rgb(255, 88, 116)">false</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Optional log to put failed JSONs into a file</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Helpful in case you are trying to debug some issue with specific ingestion failing</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">failure_log</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">enabled</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token boolean important" style="color:rgb(255, 88, 116)">false</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">log_config</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">filename</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> ./path/to/failure.json</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ingest-deploy">ingest deploy<a href="#ingest-deploy" class="hash-link" aria-label="Direct link to ingest deploy" title="Direct link to ingest deploy"></a></h3><p>The <code>ingest deploy</code> command instructs the cli to upload an ingestion recipe to DataHub to be run by DataHub's <a href="/docs/ui-ingestion">UI Ingestion</a>.
|
||
This command can also be used to schedule the ingestion while uploading or even to update existing sources. It will upload to the remote instance the
|
||
CLI is connected to, not the sink of the recipe. Use <code>datahub init</code> to set the remote if not already set.</p><p>This command will automatically create a new recipe if it doesn't exist, or update it if it does.
|
||
Note that this is a complete update, and will remove any options that were previously set.
|
||
I.e: Not specifying a schedule in the cli update command will remove the schedule from the recipe to be updated.</p><p><strong>Basic example</strong></p><p>To schedule a recipe called "Snowflake Integration", to run at 5am every day, London time with the recipe configured in a local <code>recipe.yaml</code> file:</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest deploy --name </span><span class="token string" style="color:rgb(195, 232, 141)">"Snowflake Integration"</span><span class="token plain"> --schedule </span><span class="token string" style="color:rgb(195, 232, 141)">"5 * * * *"</span><span class="token plain"> --time-zone </span><span class="token string" style="color:rgb(195, 232, 141)">"Europe/London"</span><span class="token plain"> -c recipe.yaml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>By default, the ingestion recipe's identifier is generated by hashing the name.
|
||
You can override the urn generation by passing the <code>--urn</code> flag to the CLI.</p><p><strong>Using <code>deployment</code> to avoid CLI args</strong></p><p>As an alternative to configuring settings from the CLI, all of these settings can also be set in the <code>deployment</code> field of the recipe.</p><div class="language-yml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># deployment_recipe.yml</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">deployment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"Snowflake Integration"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">schedule</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"5 * * * *"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">time_zone</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"Europe/London"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">source</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">...</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest deploy -c deployment_recipe.yml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This is particularly useful when you want all recipes to be stored in version control.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Deploy every yml recipe in a directory</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token function" style="color:rgb(130, 170, 255)">ls</span><span class="token plain"> recipe_directory/*.yml </span><span class="token operator" style="color:rgb(137, 221, 255)">|</span><span class="token plain"> </span><span class="token function" style="color:rgb(130, 170, 255)">xargs</span><span class="token plain"> -n </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"> -I </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"> datahub ingest deploy -c </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="init">init<a href="#init" class="hash-link" aria-label="Direct link to init" title="Direct link to init"></a></h3><p>The init command is used to tell <code>datahub</code> about where your DataHub instance is located. The CLI will point to localhost DataHub by default.
|
||
Running <code>datahub init</code> will allow you to customize the datahub instance you are communicating with. It has an optional <code>--use-password</code> option which allows to initialise the config using username, password. We foresee this mainly being used by admins as majority of organisations will be using SSO and there won't be any passwords to use.</p><p><strong><em>Note</em></strong>: Provide your GMS instance's host when the prompt asks you for the DataHub host.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain"># locally hosted example</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub init</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">/Users/user/.datahubenv already exists. Overwrite? [y/N]: y</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Configure which datahub instance to connect to</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Enter your DataHub host [http://localhost:8080]: http://localhost:8080</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Enter your DataHub access token []:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># acryl example</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub init</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">/Users/user/.datahubenv already exists. Overwrite? [y/N]: y</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Configure which datahub instance to connect to</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Enter your DataHub host [http://localhost:8080]: https://<your-instance-id>.acryl.io/gms</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Enter your DataHub access token []: <token generated from https://<your-instance-id>.acryl.io/settings/tokens></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="environment-variables-supported">Environment variables supported<a href="#environment-variables-supported" class="hash-link" aria-label="Direct link to Environment variables supported" title="Direct link to Environment variables supported"></a></h4><p>The environment variables listed below take precedence over the DataHub CLI config created through the <code>init</code> command.</p><ul><li><code>DATAHUB_SKIP_CONFIG</code> (default <code>false</code>) - Set to <code>true</code> to skip creating the configuration file.</li><li><code>DATAHUB_GMS_URL</code> (default <code>http://localhost:8080</code>) - Set to a URL of GMS instance</li><li><code>DATAHUB_GMS_HOST</code> (default <code>localhost</code>) - Set to a host of GMS instance. Prefer using <code>DATAHUB_GMS_URL</code> to set the URL.</li><li><code>DATAHUB_GMS_PORT</code> (default <code>8080</code>) - Set to a port of GMS instance. Prefer using <code>DATAHUB_GMS_URL</code> to set the URL.</li><li><code>DATAHUB_GMS_PROTOCOL</code> (default <code>http</code>) - Set to a protocol like <code>http</code> or <code>https</code>. Prefer using <code>DATAHUB_GMS_URL</code> to set the URL.</li><li><code>DATAHUB_GMS_TOKEN</code> (default <code>None</code>) - Used for communicating with DataHub Cloud.</li><li><code>DATAHUB_TELEMETRY_ENABLED</code> (default <code>true</code>) - Set to <code>false</code> to disable telemetry. If CLI is being run in an environment with no access to public internet then this should be disabled.</li><li><code>DATAHUB_TELEMETRY_TIMEOUT</code> (default <code>10</code>) - Set to a custom integer value to specify timeout in secs when sending telemetry.</li><li><code>DATAHUB_DEBUG</code> (default <code>false</code>) - Set to <code>true</code> to enable debug logging for CLI. Can also be achieved through <code>--debug</code> option of the CLI. This exposes sensitive information in logs, enabling on production instances should be avoided especially if UI ingestion is in use as logs can be made available for runs through the UI.</li><li><code>DATAHUB_VERSION</code> (default <code>head</code>) - Set to a specific version to run quickstart with the particular version of docker images.</li><li><code>ACTIONS_VERSION</code> (default <code>head</code>) - Set to a specific version to run quickstart with that image tag of <code>datahub-actions</code> container.</li><li><code>DATAHUB_ACTIONS_IMAGE</code> (default <code>acryldata/datahub-actions</code>) - Set to <code>-slim</code> to run a slimmer actions container without pyspark/deequ features.</li></ul><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token assign-left variable" style="color:rgb(191, 199, 213)">DATAHUB_SKIP_CONFIG</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">false</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token assign-left variable" style="color:rgb(191, 199, 213)">DATAHUB_GMS_URL</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">http://localhost:8080</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token assign-left variable" style="color:rgb(191, 199, 213)">DATAHUB_GMS_TOKEN</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token assign-left variable" style="color:rgb(191, 199, 213)">DATAHUB_TELEMETRY_ENABLED</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">true</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token assign-left variable" style="color:rgb(191, 199, 213)">DATAHUB_TELEMETRY_TIMEOUT</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token assign-left variable" style="color:rgb(191, 199, 213)">DATAHUB_DEBUG</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">false</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="container">container<a href="#container" class="hash-link" aria-label="Direct link to container" title="Direct link to container"></a></h3><p>A group of commands to interact with containers in DataHub.</p><p>e.g. You can use this to apply a tag to all datasets recursively in this container.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub container tag --container-urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:container:0e9e46bd6d5cf645f33d5a8f0254bc2d"</span><span class="token plain"> --tag-urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:tag:tag1"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub container domain --container-urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:container:3f2effd1fbe154a4d60b597263a41e41"</span><span class="token plain"> --domain-urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:domain:ajsajo-b832-4ab3-8881-7ed5e991a44c"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub container owner --container-urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:container:3f2effd1fbe154a4d60b597263a41e41"</span><span class="token plain"> --owner-urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:corpGroup:eng@example.com"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub container term --container-urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:container:3f2effd1fbe154a4d60b597263a41e41"</span><span class="token plain"> --term-urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:term:PII"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="check">check<a href="#check" class="hash-link" aria-label="Direct link to check" title="Direct link to check"></a></h3><p>The datahub package is composed of different plugins that allow you to connect to different metadata sources and ingest metadata from them.
|
||
The <code>check</code> command allows you to check if all plugins are loaded correctly as well as validate an individual MCE-file.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="restore-indices">restore-indices<a href="#restore-indices" class="hash-link" aria-label="Direct link to restore-indices" title="Direct link to restore-indices"></a></h4><p>This command allows you to restore indices for one or more <code>urn</code>.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub --debug check restore-indices --urn </span><span class="token string" style="color:rgb(195, 232, 141)">"URN"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>It can also take <code>--file</code> argument that points to a file that has list of urns like</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub check restore-indices --file ./urn.txt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>where urn.txt is like this</p><div class="language-urn.txt codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-urn.txt codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.schema1.test_all_nulls,PROD)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">urn:li:dataset:(urn:li:dataPlatform:platform1,test_db.schema2.test_complex_types,PROD)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">urn:li:dataset:(urn:li:dataPlatform:redshift,test_db.schema3.test_few_rows,PROD)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="get-kafka-consumer-offsets">get-kafka-consumer-offsets<a href="#get-kafka-consumer-offsets" class="hash-link" aria-label="Direct link to get-kafka-consumer-offsets" title="Direct link to get-kafka-consumer-offsets"></a></h4><p>This required DataHub Cloud <code>0.3.12.x</code> or above version.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub check get-kafka-consumer-offsets</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>which can give can print out the details of lag in kafka topics.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">{'mcl': {'generic-mae-consumer-job-client': {'MetadataChangeLog_Versioned_v1': {'metrics': {'avgLag': 36,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'maxLag': 36,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'medianLag': 36,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'totalLag': 36},</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'partitions': {'0': {'lag': 36,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'offset': 257318}}}}},</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'mcl-timeseries': {'generic-mae-consumer-job-client': {'MetadataChangeLog_Timeseries_v1': {'metrics': {'avgLag': 0,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'maxLag': 0,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'medianLag': 0,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'totalLag': 0},</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'partitions': {'0': {'lag': 0,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'offset': 113}}}}},</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'mcp': {'generic-mce-consumer-job-client': {'MetadataChangeProposal_v1': {'metrics': {'avgLag': 7222149,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'maxLag': 7222149,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'medianLag': 7222149,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'totalLag': 7222149},</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'partitions': {'0': {'lag': 7222149,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> 'offset': 250254}}}}}}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="delete">delete<a href="#delete" class="hash-link" aria-label="Direct link to delete" title="Direct link to delete"></a></h3><p>The <code>delete</code> command allows you to delete metadata from DataHub.</p><p>The <a href="/docs/how/delete-metadata">metadata deletion guide</a> covers the various options for the delete command.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="exists">exists<a href="#exists" class="hash-link" aria-label="Direct link to exists" title="Direct link to exists"></a></h3><p>The exists command can be used to check if an entity exists in DataHub.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> datahub exists --urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token boolean" style="color:rgb(255, 88, 116)">true</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> datahub exists --urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataset:(urn:li:dataPlatform:hive,NonExistentHiveDataset,PROD)"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token boolean" style="color:rgb(255, 88, 116)">false</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="get">get<a href="#get" class="hash-link" aria-label="Direct link to get" title="Direct link to get"></a></h3><p>The <code>get</code> command allows you to easily retrieve metadata from DataHub, by using the REST API. This works for both versioned aspects and timeseries aspects. For timeseries aspects, it fetches the latest value.
|
||
For example the following command gets the ownership aspect from the dataset <code>urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)</code></p><div class="language-shell-session codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell-session codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token command shell-symbol important">$</span><span class="token command"> </span><span class="token command bash language-bash">datahub get --urn </span><span class="token command bash language-bash string" style="color:rgb(195, 232, 141)">"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)"</span><span class="token command bash language-bash"> --aspect ownership</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token output">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "ownership": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "lastModified": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "actor": "urn:li:corpuser:jdoe",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "time": 1680210917580</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "owners": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "owner": "urn:li:corpuser:jdoe",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "source": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "type": "SERVICE"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "type": "NONE"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="put">put<a href="#put" class="hash-link" aria-label="Direct link to put" title="Direct link to put"></a></h3><p>The <code>put</code> group of commands allows you to write metadata into DataHub. This is a flexible way for you to issue edits to metadata from the command line.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="put-aspect">put aspect<a href="#put-aspect" class="hash-link" aria-label="Direct link to put aspect" title="Direct link to put aspect"></a></h4><p>The <strong>put aspect</strong> (also the default <code>put</code>) command instructs <code>datahub</code> to set a specific aspect for an entity to a specified value.
|
||
For example, the command shown below sets the <code>ownership</code> aspect of the dataset <code>urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)</code> to the value in the file <code>ownership.json</code>.
|
||
The JSON in the <code>ownership.json</code> file needs to conform to the <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl" target="_blank" rel="noopener noreferrer"><code>Ownership</code></a> Aspect model as shown below.</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"owners"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"owner"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:corpuser:jdoe"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"DEVELOPER"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"owner"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:corpuser:jdub"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"DATAOWNER"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub --debug put --urn "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" --aspect ownership -d ownership.json</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[DATE_TIMESTAMP] DEBUG {datahub.cli.cli_utils:340} - Attempting to emit to DataHub GMS; using curl equivalent to:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">curl -X POST -H 'User-Agent: python-requests/2.26.0' -H 'Accept-Encoding: gzip, deflate' -H 'Accept: */*' -H 'Connection: keep-alive' -H 'X-RestLi-Protocol-Version: 2.0.0' -H 'Content-Type: application/json' --data '{"proposal": {"entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", "aspectName": "ownership", "changeType": "UPSERT", "aspect": {"contentType": "application/json", "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jdoe\", \"type\": \"DEVELOPER\"}, {\"owner\": \"urn:li:corpuser:jdub\", \"type\": \"DATAOWNER\"}]}"}}}' 'http://localhost:8080/aspects/?action=ingestProposal'</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Update succeeded with status 200</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="put-platform">put platform<a href="#put-platform" class="hash-link" aria-label="Direct link to put platform" title="Direct link to put platform"></a></h4><p><strong>🤝 Version Compatibility:</strong> <code>acryl-datahub>0.8.44.4</code></p><p>The <strong>put platform</strong> command instructs <code>datahub</code> to create or update metadata about a data platform. This is very useful if you are using a custom data platform, to set up its logo and display name for a native UI experience.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub put platform --name longtail_schemas --display_name </span><span class="token string" style="color:rgb(195, 232, 141)">"Long Tail Schemas"</span><span class="token plain"> --logo </span><span class="token string" style="color:rgb(195, 232, 141)">"https://flink.apache.org/img/logo/png/50/color_50.png"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">✅ Successfully wrote data platform metadata </span><span class="token keyword" style="font-style:italic">for</span><span class="token plain"> urn:li:dataPlatform:longtail_schemas to DataHub </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">DataHubRestEmitter: configured to talk to https://longtailcompanions.acryl.io/api/gms with token: eyJh**********Cics</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timeline">timeline<a href="#timeline" class="hash-link" aria-label="Direct link to timeline" title="Direct link to timeline"></a></h3><p>The <code>timeline</code> command allows you to view a version history for entities. Currently only supported for Datasets. For example,
|
||
the following command will show you the modifications to tags for a dataset for the past week. The output includes a computed semantic version,
|
||
relevant for schema changes only currently, the target of the modification, and a description of the change including a timestamp.
|
||
The default output is sanitized to be more readable, but the full API output can be obtained by passing the <code>--verbose</code> flag and
|
||
to get the raw JSON difference in addition to the API output you can add the <code>--raw</code> flag. For more details about the feature please see <a href="/docs/dev-guides/timeline">the main feature page</a></p><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub timeline --urn "urn:li:dataset:(urn:li:dataPlatform:mysql,User.UserAccount,PROD)" --category TAG --start 7daysago</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">2022-02-17 14:03:42 - 0.0.0-computed</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> MODIFY TAG dataset:mysql:User.UserAccount : A change in aspect editableSchemaMetadata happened at time 2022-02-17 20:03:42.0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">2022-02-17 14:17:30 - 0.0.0-computed</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> MODIFY TAG dataset:mysql:User.UserAccount : A change in aspect editableSchemaMetadata happened at time 2022-02-17 20:17:30.118</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="entity-specific-commands">Entity Specific Commands<a href="#entity-specific-commands" class="hash-link" aria-label="Direct link to Entity Specific Commands" title="Direct link to Entity Specific Commands"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dataset-dataset-entity">dataset (Dataset Entity)<a href="#dataset-dataset-entity" class="hash-link" aria-label="Direct link to dataset (Dataset Entity)" title="Direct link to dataset (Dataset Entity)"></a></h3><p>The <code>dataset</code> command allows you to interact with Dataset entities in DataHub, including creating, updating, retrieving, and validating Dataset metadata.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Get a dataset and write to YAML file</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub dataset get --urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataset:(urn:li:dataPlatform:hive,example_table,PROD)"</span><span class="token plain"> --to-file dataset.yaml</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Create or update dataset from YAML file</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub dataset upsert -f dataset.yaml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>➡️ <a href="/docs/cli-commands/dataset">Learn more about the dataset command</a></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="user-user-entity">user (User Entity)<a href="#user-user-entity" class="hash-link" aria-label="Direct link to user (User Entity)" title="Direct link to user (User Entity)"></a></h3><p>The <code>user</code> command allows you to interact with the User entity.
|
||
It currently supports the <code>upsert</code> operation, which can be used to create a new user or update an existing one.
|
||
For detailed information, please refer to <a href="/docs/api/tutorials/owners#upsert-users">Creating Users and Groups with Datahub CLI</a>.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub user upsert -f users.yaml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>An example of <code>users.yaml</code> would look like as in <a href="https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/cli_usage/user/bar.user.dhub.yaml" target="_blank" rel="noopener noreferrer">bar.user.dhub.yaml</a> file for the complete code.</p><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">id</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> bar@acryl.io</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">first_name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> The</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">last_name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> Bar</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">email</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> bar@acryl.io</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">slack</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"@the_bar_raiser"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">description</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"I like raising the bar higher"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">groups</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> foogroup@acryl.io</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">id</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> datahub</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">slack</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"@datahubproject"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">phone</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"1-800-GOT-META"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">description</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"The DataHub Project"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">picture_link</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/datahub-logo-color-stable.svg"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="group-group-entity">group (Group Entity)<a href="#group-group-entity" class="hash-link" aria-label="Direct link to group (Group Entity)" title="Direct link to group (Group Entity)"></a></h3><p>The <code>group</code> command allows you to interact with the Group entity.
|
||
It currently supports the <code>upsert</code> operation, which can be used to create a new group or update an existing one with embedded Users.
|
||
For more information, please refer to <a href="/docs/api/tutorials/owners#upsert-users">Creating Users and Groups with Datahub CLI</a>.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub group upsert -f group.yaml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>An example of <code>group.yaml</code> would look like as in <a href="https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/cli_usage/group/foo.group.dhub.yaml" target="_blank" rel="noopener noreferrer">foo.group.dhub.yaml</a> file for the complete code.</p><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token key atrule">id</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> foogroup@acryl.io</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">display_name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> Foo Group</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">admins</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> datahub</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">members</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> bar@acryl.io </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># refer to a user either by id or by urn</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">id</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> joe@acryl.io </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># inline specification of user</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">slack</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"@joe_shmoe"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">display_name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"Joe's Hub"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dataproduct-data-product-entity">dataproduct (Data Product Entity)<a href="#dataproduct-data-product-entity" class="hash-link" aria-label="Direct link to dataproduct (Data Product Entity)" title="Direct link to dataproduct (Data Product Entity)"></a></h3><p><strong>🤝 Version Compatibility:</strong> <code>acryl-datahub>=0.10.2.4</code></p><p>The dataproduct group of commands allows you to manage the lifecycle of a DataProduct entity on DataHub.
|
||
See the <a href="/docs/dataproducts">Data Products</a> page for more details on what a Data Product is and how DataHub represents it.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub dataproduct --help</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Commands:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> upsert* Upsert attributes to a Data Product </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> DataHub</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> update Create or Update a Data Product </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> add_asset Add an asset to a Data Product</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> add_owner Add an owner to a Data Product</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> delete Delete a Data Product </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token function" style="color:rgb(130, 170, 255)">diff</span><span class="token plain"> Diff a Data Product </span><span class="token function" style="color:rgb(130, 170, 255)">file</span><span class="token plain"> with its twin </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> DataHub</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> get Get a Data Product from DataHub</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> remove_asset Add an asset to a Data Product</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> remove_owner Remove an owner from a Data Product</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> set_description Set description </span><span class="token keyword" style="font-style:italic">for</span><span class="token plain"> a Data Product </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> DataHub</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Here we detail the sub-commands available under the dataproduct group of commands:</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="upsert">upsert<a href="#upsert" class="hash-link" aria-label="Direct link to upsert" title="Direct link to upsert"></a></h4><p>Use this to upsert a data product yaml file into DataHub. This will create the data product if it doesn't exist already. Remember, this will upsert all the fields that are specified in the yaml file and will not touch the fields that are not specified. For example, if you do not specify the <code>description</code> field in the yaml file, then <code>upsert</code> will not modify the description field on the Data Product entity in DataHub. To keep this file sync-ed with the metadata on DataHub use the <a href="#diff">diff</a> command. The format of the yaml file is available <a href="/docs/dataproducts#creating-a-data-product-yaml--git">here</a>.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Usage</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> datahub dataproduct upsert -f data_product.yaml</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="update">update<a href="#update" class="hash-link" aria-label="Direct link to update" title="Direct link to update"></a></h4><p>Use this to fully replace a data product's metadata in DataHub from a yaml file. This will create the data product if it doesn't exist already. Remember, this will update all the fields including ones that are not specified in the yaml file. For example, if you do not specify the <code>description</code> field in the yaml file, then <code>update</code> will set the description field on the Data Product entity in DataHub to empty. To keep this file sync-ed with the metadata on DataHub use the <a href="#diff">diff</a> command. The format of the yaml file is available <a href="/docs/dataproducts#creating-a-data-product-yaml--git">here</a>.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Usage</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> datahub dataproduct upsert -f data_product.yaml</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>❗<strong>Pro-Tip: upsert versus update</strong></p><p>Wondering which command is right for you? Use <code>upsert</code> if there are certain elements of metadata that you don't want to manage using the yaml file (e.g. owners, assets or description). Use <code>update</code> if you want to manage the entire data product's metadata using the yaml file.</p></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="diff">diff<a href="#diff" class="hash-link" aria-label="Direct link to diff" title="Direct link to diff"></a></h4><p>Use this to keep a data product yaml file updated from its server-side version in DataHub.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Usage</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> datahub dataproduct </span><span class="token function" style="color:rgb(130, 170, 255)">diff</span><span class="token plain"> -f data_product.yaml --update</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="get-1">get<a href="#get-1" class="hash-link" aria-label="Direct link to get" title="Direct link to get"></a></h4><p>Use this to get a data product entity from DataHub and optionally write it to a yaml file</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Usage</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> datahub dataproduct get --urn urn:li:dataProduct:pet_of_the_week --to-file pet_of_the_week_dataproduct.yaml</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"id"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataProduct:pet_of_the_week"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"domain"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:domain:dcadded3-2b70-4679-8b28-02ac9abc92eb"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"assets"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.pet_details,PROD)"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dashboard:(looker,dashboards.19)"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataFlow:(airflow,snowflake_load,prod)"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"display_name"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"Pet of the Week Campaign"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"owners"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"id"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:corpuser:jdoe"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"type"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"BUSINESS_OWNER"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"description"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"This campaign includes Pet of the Week data."</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"tags"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:tag:adoption"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"terms"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:glossaryTerm:ClientsAndAccounts.AccountBalance"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"properties"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"lifecycle"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"production"</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"sla"</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"7am every day"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Data Product yaml written to pet_of_the_week_dataproduct.yaml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="add_asset">add_asset<a href="#add_asset" class="hash-link" aria-label="Direct link to add_asset" title="Direct link to add_asset"></a></h4><p>Use this to add a data asset to a Data Product.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Usage</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> datahub dataproduct add_asset --urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataProduct:pet_of_the_week"</span><span class="token plain"> --asset </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="remove_asset">remove_asset<a href="#remove_asset" class="hash-link" aria-label="Direct link to remove_asset" title="Direct link to remove_asset"></a></h4><p>Use this to remove a data asset from a Data Product.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Usage</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> datahub dataproduct remove_asset --urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataProduct:pet_of_the_week"</span><span class="token plain"> --asset </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="add_owner">add_owner<a href="#add_owner" class="hash-link" aria-label="Direct link to add_owner" title="Direct link to add_owner"></a></h4><p>Use this to add an owner to a Data Product.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Usage</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> datahub dataproduct add_owner --urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataProduct:pet_of_the_week"</span><span class="token plain"> --owner </span><span class="token string" style="color:rgb(195, 232, 141)">"jdoe@longtail.com"</span><span class="token plain"> --owner-type BUSINESS_OWNER</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="remove_owner">remove_owner<a href="#remove_owner" class="hash-link" aria-label="Direct link to remove_owner" title="Direct link to remove_owner"></a></h4><p>Use this to remove an owner from a Data Product.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Usage</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> datahub dataproduct remove_owner --urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataProduct:pet_of_the_week"</span><span class="token plain"> --owner </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:corpUser:jdoe@longtail.com"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="set_description">set_description<a href="#set_description" class="hash-link" aria-label="Direct link to set_description" title="Direct link to set_description"></a></h4><p>Use this to attach rich documentation for a Data Product in DataHub.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> datahub dataproduct set_description --urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataProduct:pet_of_the_week"</span><span class="token plain"> --description </span><span class="token string" style="color:rgb(195, 232, 141)">"This is the pet dataset"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># For uploading rich documentation from a markdown file, use the --md-file option</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># > datahub dataproduct set_description --urn "urn:li:dataProduct:pet_of_the_week" --md-file ./pet_of_the_week.md</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="delete-1">delete<a href="#delete-1" class="hash-link" aria-label="Direct link to delete" title="Direct link to delete"></a></h4><p>Use this to delete a Data Product from DataHub. Default to <code>--soft</code> which preserves metadata, use <code>--hard</code> to erase all metadata associated with this Data Product.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> datahub dataproduct delete --urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataProduct:pet_of_the_week"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># For Hard Delete see below:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># > datahub dataproduct delete --urn "urn:li:dataProduct:pet_of_the_week" --hard</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="miscellaneous-admin-commands">Miscellaneous Admin Commands<a href="#miscellaneous-admin-commands" class="hash-link" aria-label="Direct link to Miscellaneous Admin Commands" title="Direct link to Miscellaneous Admin Commands"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lite-experimental">lite (experimental)<a href="#lite-experimental" class="hash-link" aria-label="Direct link to lite (experimental)" title="Direct link to lite (experimental)"></a></h3><p>The lite group of commands allow you to run an embedded, lightweight DataHub instance for command line exploration of your metadata. This is intended more for developer tool oriented usage rather than as a production server instance for DataHub. See <a href="/docs/datahub_lite">DataHub Lite</a> for more information about how you can ingest metadata into DataHub Lite and explore your metadata easily.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="telemetry">telemetry<a href="#telemetry" class="hash-link" aria-label="Direct link to telemetry" title="Direct link to telemetry"></a></h3><p>To help us understand how people are using DataHub, we collect anonymous usage statistics on actions such as command invocations via Mixpanel.
|
||
We do not collect private information such as IP addresses, contents of ingestions, or credentials.
|
||
The code responsible for collecting and broadcasting these events is open-source and can be found <a href="https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/telemetry/telemetry.py" target="_blank" rel="noopener noreferrer">within our GitHub</a>.</p><p>Telemetry is enabled by default, and the <code>telemetry</code> command lets you toggle the sending of these statistics via <code>telemetry enable/disable</code>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="migrate">migrate<a href="#migrate" class="hash-link" aria-label="Direct link to migrate" title="Direct link to migrate"></a></h3><p>The <code>migrate</code> group of commands allows you to perform certain kinds of migrations.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="dataplatform2instance">dataplatform2instance<a href="#dataplatform2instance" class="hash-link" aria-label="Direct link to dataplatform2instance" title="Direct link to dataplatform2instance"></a></h4><p>The <code>dataplatform2instance</code> migration command allows you to migrate your entities from an instance-agnostic platform identifier to an instance-specific platform identifier. If you have ingested metadata in the past for this platform and would like to transfer any important metadata over to the new instance-specific entities, then you should use this command. For example, if your users have added documentation or added tags or terms to your datasets, then you should run this command to transfer this metadata over to the new entities. For further context, read the Platform Instance Guide <a href="/docs/platform-instances">here</a>.</p><p>A few important options worth calling out:</p><ul><li>--dry-run / -n : Use this to get a report for what will be migrated before running</li><li>--force / -F : Use this if you know what you are doing and do not want to get a confirmation prompt before migration is started</li><li>--keep : When enabled, will preserve the old entities and not delete them. Default behavior is to soft-delete old entities.</li><li>--hard : When enabled, will hard-delete the old entities.</li></ul><p><strong><em>Note</em></strong>: Timeseries aspects such as Usage Statistics and Dataset Profiles are not migrated over to the new entity instances, you will get new data points created when you re-run ingestion using the <code>usage</code> or sources with profiling turned on.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="dry-run">Dry Run<a href="#dry-run" class="hash-link" aria-label="Direct link to Dry Run" title="Direct link to Dry Run"></a></h5><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub migrate dataplatform2instance --platform elasticsearch --instance prod_index --dry-run</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Starting migration: platform:elasticsearch, instance=prod_index, force=False, dry-run=True</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">100% (25 of 25) |####################################################################################################################################################################################| Elapsed Time: 0:00:00 Time: 0:00:00</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[Dry Run] Migration Report:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--------------</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[Dry Run] Migration Run Id: migrate-5710349c-1ec7-4b83-a7d3-47d71b7e972e</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[Dry Run] Num entities created = 25</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[Dry Run] Num entities affected = 0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[Dry Run] Num entities migrated = 25</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[Dry Run] Details:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[Dry Run] New Entities Created: {'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.datahubretentionindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.schemafieldindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.system_metadata_service_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.tagindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dataset_datasetprofileaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.mlmodelindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.mlfeaturetableindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.datajob_datahubingestioncheckpointaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.datahub_usage_event,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dataset_operationaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.datajobindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dataprocessindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.glossarytermindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dataplatformindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.mlmodeldeploymentindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.datajob_datahubingestionrunsummaryaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.graph_service_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.datahubpolicyindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dataset_datasetusagestatisticsaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dashboardindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.glossarynodeindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.mlfeatureindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dataflowindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.mlprimarykeyindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.chartindex_v2,PROD)'}</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[Dry Run] External Entities Affected: None</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[Dry Run] Old Entities Migrated = {'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dataset_datasetusagestatisticsaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,mlmodelindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,mlmodeldeploymentindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,datajob_datahubingestionrunsummaryaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,datahubretentionindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,datahubpolicyindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dataset_datasetprofileaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,glossarynodeindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dataset_operationaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,graph_service_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,datajobindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,mlprimarykeyindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dashboardindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,datajob_datahubingestioncheckpointaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,tagindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,datahub_usage_event,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,schemafieldindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,mlfeatureindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dataprocessindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dataplatformindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,mlfeaturetableindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,glossarytermindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dataflowindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,chartindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,system_metadata_service_v1,PROD)'}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h5 class="anchor anchorWithStickyNavbar_LWe7" id="real-migration-with-soft-delete">Real Migration (with soft-delete)<a href="#real-migration-with-soft-delete" class="hash-link" aria-label="Direct link to Real Migration (with soft-delete)" title="Direct link to Real Migration (with soft-delete)"></a></h5><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">> datahub migrate dataplatform2instance --platform hive --instance</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub migrate dataplatform2instance --platform hive --instance warehouse</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Starting migration: platform:hive, instance=warehouse, force=False, dry-run=False</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Will migrate 4 urns such as ['urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)']</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">New urns will look like ['urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.logging_events,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.fct_users_created,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.SampleHiveDataset,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.fct_users_deleted,PROD)']</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Ok to proceed? [y/N]:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Migration Report:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--------------</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Migration Run Id: migrate-f5ae7201-4548-4bee-aed4-35758bb78c89</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Num entities created = 4</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Num entities affected = 0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Num entities migrated = 4</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Details:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">New Entities Created: {'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.SampleHiveDataset,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.fct_users_deleted,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.logging_events,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.fct_users_created,PROD)'}</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">External Entities Affected: None</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Old Entities Migrated = {'urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)'}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="alternate-installation-options">Alternate Installation Options<a href="#alternate-installation-options" class="hash-link" aria-label="Direct link to Alternate Installation Options" title="Direct link to Alternate Installation Options"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="using-docker">Using docker<a href="#using-docker" class="hash-link" aria-label="Direct link to Using docker" title="Direct link to Using docker"></a></h3><p><a href="https://hub.docker.com/r/acryldata/datahub-ingestion" target="_blank" rel="noopener noreferrer"><img loading="lazy" src="https://img.shields.io/docker/pulls/acryldata/datahub-ingestion?style=plastic" alt="Docker Hub" class="img_ev3q"></a></p><p>If you don't want to install locally, you can alternatively run metadata ingestion within a Docker container.
|
||
We have prebuilt images available on <a href="https://hub.docker.com/r/acryldata/datahub-ingestion" target="_blank" rel="noopener noreferrer">Docker hub</a>. All plugins will be installed and enabled automatically.</p><p>You can use the <code>datahub-ingestion</code> docker image as explained in <a href="/docs/docker">Docker Images</a>. In case you are using Kubernetes you can start a pod with the <code>datahub-ingestion</code> docker image, log onto a shell on the pod and you should have the access to datahub CLI in your kubernetes cluster.</p><p><em>Limitation: the datahub_docker.sh convenience script assumes that the recipe and any input/output files are accessible in the current working directory or its subdirectories. Files outside the current working directory will not be found, and you'll need to invoke the Docker image directly.</em></p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Assumes the DataHub repo is cloned locally.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">./metadata-ingestion/scripts/datahub_docker.sh ingest -c ./examples/recipes/example_to_datahub_rest.yml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-from-source">Install from source<a href="#install-from-source" class="hash-link" aria-label="Direct link to Install from source" title="Direct link to Install from source"></a></h3><p>If you'd like to install from source, see the <a href="/docs/metadata-ingestion/developing">developer guide</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="installing-plugins">Installing Plugins<a href="#installing-plugins" class="hash-link" aria-label="Direct link to Installing Plugins" title="Direct link to Installing Plugins"></a></h2><p>We use a plugin architecture so that you can install only the dependencies you actually need. Click the plugin name to learn more about the specific source recipe and any FAQs!</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sources">Sources<a href="#sources" class="hash-link" aria-label="Direct link to Sources" title="Direct link to Sources"></a></h3><p>Please see our <a href="/integrations">Integrations page</a> if you want to filter on the features offered by each source.</p><table><thead><tr><th>Plugin Name</th><th>Install Command</th><th>Provides</th></tr></thead><tbody><tr><td><a href="/docs/generated/ingestion/sources/metadata-file">metadata-file</a></td><td><em>included by default</em></td><td>File source and sink</td></tr><tr><td><a href="/docs/generated/ingestion/sources/athena">athena</a></td><td><code>pip install 'acryl-datahub[athena]'</code></td><td>AWS Athena source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/bigquery">bigquery</a></td><td><code>pip install 'acryl-datahub[bigquery]'</code></td><td>BigQuery source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/file-based-lineage">datahub-lineage-file</a></td><td><em>no additional dependencies</em></td><td>Lineage File source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/business-glossary">datahub-business-glossary</a></td><td><em>no additional dependencies</em></td><td>Business Glossary File source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/dbt">dbt</a></td><td><code>pip install 'acryl-datahub[dbt]'</code></td><td>dbt source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/dremio">dremio</a></td><td><code>pip install 'acryl-datahub[dremio]'</code></td><td>Dremio Source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/druid">druid</a></td><td><code>pip install 'acryl-datahub[druid]'</code></td><td>Druid Source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/feast">feast</a></td><td><code>pip install 'acryl-datahub[feast]'</code></td><td>Feast source (0.26.0)</td></tr><tr><td><a href="/docs/generated/ingestion/sources/glue">glue</a></td><td><code>pip install 'acryl-datahub[glue]'</code></td><td>AWS Glue source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/hana">hana</a></td><td><code>pip install 'acryl-datahub[hana]'</code></td><td>SAP HANA source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/hive">hive</a></td><td><code>pip install 'acryl-datahub[hive]'</code></td><td>Hive source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/kafka">kafka</a></td><td><code>pip install 'acryl-datahub[kafka]'</code></td><td>Kafka source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/kafka-connect">kafka-connect</a></td><td><code>pip install 'acryl-datahub[kafka-connect]'</code></td><td>Kafka connect source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/ldap">ldap</a></td><td><code>pip install 'acryl-datahub[ldap]'</code> (<a href="https://www.python-ldap.org/en/python-ldap-3.3.0/installing.html#build-prerequisites" target="_blank" rel="noopener noreferrer">extra requirements</a>)</td><td>LDAP source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/looker">looker</a></td><td><code>pip install 'acryl-datahub[looker]'</code></td><td>Looker source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/looker#module-lookml">lookml</a></td><td><code>pip install 'acryl-datahub[lookml]'</code></td><td>LookML source, requires Python 3.7+</td></tr><tr><td><a href="/docs/generated/ingestion/sources/metabase">metabase</a></td><td><code>pip install 'acryl-datahub[metabase]'</code></td><td>Metabase source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/mode">mode</a></td><td><code>pip install 'acryl-datahub[mode]'</code></td><td>Mode Analytics source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/mongodb">mongodb</a></td><td><code>pip install 'acryl-datahub[mongodb]'</code></td><td>MongoDB source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/mssql">mssql</a></td><td><code>pip install 'acryl-datahub[mssql]'</code></td><td>SQL Server source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/mysql">mysql</a></td><td><code>pip install 'acryl-datahub[mysql]'</code></td><td>MySQL source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/mariadb">mariadb</a></td><td><code>pip install 'acryl-datahub[mariadb]'</code></td><td>MariaDB source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/openapi">openapi</a></td><td><code>pip install 'acryl-datahub[openapi]'</code></td><td>OpenApi Source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/oracle">oracle</a></td><td><code>pip install 'acryl-datahub[oracle]'</code></td><td>Oracle source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/postgres">postgres</a></td><td><code>pip install 'acryl-datahub[postgres]'</code></td><td>Postgres source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/redash">redash</a></td><td><code>pip install 'acryl-datahub[redash]'</code></td><td>Redash source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/redshift">redshift</a></td><td><code>pip install 'acryl-datahub[redshift]'</code></td><td>Redshift source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/sagemaker">sagemaker</a></td><td><code>pip install 'acryl-datahub[sagemaker]'</code></td><td>AWS SageMaker source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/salesforce">salesforce</a></td><td><code>pip install 'acryl-datahub[salesforce]'</code></td><td>Salesforce source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/snowflake">snowflake</a></td><td><code>pip install 'acryl-datahub[snowflake]'</code></td><td>Snowflake source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/sqlalchemy">sqlalchemy</a></td><td><code>pip install 'acryl-datahub[sqlalchemy]'</code></td><td>Generic SQLAlchemy source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/superset">superset</a></td><td><code>pip install 'acryl-datahub[superset]'</code></td><td>Superset source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/tableau">tableau</a></td><td><code>pip install 'acryl-datahub[tableau]'</code></td><td>Tableau source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/trino">trino</a></td><td><code>pip install 'acryl-datahub[trino]'</code></td><td>Trino source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/trino">starburst-trino-usage</a></td><td><code>pip install 'acryl-datahub[starburst-trino-usage]'</code></td><td>Starburst Trino usage statistics source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/nifi">nifi</a></td><td><code>pip install 'acryl-datahub[nifi]'</code></td><td>NiFi source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/powerbi#module-powerbi">powerbi</a></td><td><code>pip install 'acryl-datahub[powerbi]'</code></td><td>Microsoft Power BI source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/powerbi#module-powerbi-report-server">powerbi-report-server</a></td><td><code>pip install 'acryl-datahub[powerbi-report-server]'</code></td><td>Microsoft Power BI Report Server source</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sinks">Sinks<a href="#sinks" class="hash-link" aria-label="Direct link to Sinks" title="Direct link to Sinks"></a></h3><table><thead><tr><th>Plugin Name</th><th>Install Command</th><th>Provides</th></tr></thead><tbody><tr><td><a href="/docs/metadata-ingestion/sink_docs/metadata-file">metadata-file</a></td><td><em>included by default</em></td><td>File source and sink</td></tr><tr><td><a href="/docs/metadata-ingestion/sink_docs/console">console</a></td><td><em>included by default</em></td><td>Console sink</td></tr><tr><td><a href="/docs/metadata-ingestion/sink_docs/datahub">datahub-rest</a></td><td><code>pip install 'acryl-datahub[datahub-rest]'</code></td><td>DataHub sink over REST API</td></tr><tr><td><a href="/docs/metadata-ingestion/sink_docs/datahub">datahub-kafka</a></td><td><code>pip install 'acryl-datahub[datahub-kafka]'</code></td><td>DataHub sink over Kafka</td></tr></tbody></table><p>These plugins can be mixed and matched as desired. For example:</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">pip </span><span class="token function" style="color:rgb(130, 170, 255)">install</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">'acryl-datahub[bigquery,datahub-rest]'</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="check-the-active-plugins">Check the active plugins<a href="#check-the-active-plugins" class="hash-link" aria-label="Direct link to Check the active plugins" title="Direct link to Check the active plugins"></a></h3><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub check plugins</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="release-notes-and-cli-versions">Release Notes and CLI versions<a href="#release-notes-and-cli-versions" class="hash-link" aria-label="Direct link to Release Notes and CLI versions" title="Direct link to Release Notes and CLI versions"></a></h2><p>The server release notes can be found in <a href="https://github.com/datahub-project/datahub/releases" target="_blank" rel="noopener noreferrer">github releases</a>. These releases are done approximately every week on a regular cadence unless a blocking issue or regression is discovered.</p><p>CLI release is made through a different repository and release notes can be found in <a href="https://github.com/acryldata/datahub/releases" target="_blank" rel="noopener noreferrer">acryldata releases</a>. At least one release which is tied to the server release is always made alongwith the server release. Multiple other bigfix releases are made in between based on amount of fixes that are merged between the server release mentioned above.</p><p>If server with version <code>0.8.28</code> is being used then CLI used to connect to it should be <code>0.8.28.x</code>. Tests of new CLI are not ran with older server versions so it is not recommended to update the CLI if the server is not updated.</p></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="slackUtm_uoBr"><div class="slackUtm_uoBr"><hr>Need more help? Join the conversation in <a href="https://datahub.com/slack?utm_source=docs&utm_medium=footer&utm_campaign=docs_footer&utm_content=docs/cli">Slack!</a></div></div><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/datahub-project/datahub/blob/master/docs/cli.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_VsjB"></div></div></footer><div class="feedbackWrapper_mUHF"><div class="feedbackWidget_PX4d"><div class="feedbackButtons_wn3V"><strong>Is this page helpful?</strong><div><button class="feedbackButton_UgQs"><span role="img" aria-label="like" class="anticon anticon-like"><svg viewBox="64 64 896 896" focusable="false" data-icon="like" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M885.9 533.7c16.8-22.2 26.1-49.4 26.1-77.7 0-44.9-25.1-87.4-65.5-111.1a67.67 67.67 0 00-34.3-9.3H572.4l6-122.9c1.4-29.7-9.1-57.9-29.5-79.4A106.62 106.62 0 00471 99.9c-52 0-98 35-111.8 85.1l-85.9 311H144c-17.7 0-32 14.3-32 32v364c0 17.7 14.3 32 32 32h601.3c9.2 0 18.2-1.8 26.5-5.4 47.6-20.3 78.3-66.8 78.3-118.4 0-12.6-1.8-25-5.4-37 16.8-22.2 26.1-49.4 26.1-77.7 0-12.6-1.8-25-5.4-37 16.8-22.2 26.1-49.4 26.1-77.7-.2-12.6-2-25.1-5.6-37.1zM184 852V568h81v284h-81zm636.4-353l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 16.5-7.2 32.2-19.6 43l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 16.5-7.2 32.2-19.6 43l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 22.4-13.2 42.6-33.6 51.8H329V564.8l99.5-360.5a44.1 44.1 0 0142.2-32.3c7.6 0 15.1 2.2 21.1 6.7 9.9 7.4 15.2 18.6 14.6 30.5l-9.6 198.4h314.4C829 418.5 840 436.9 840 456c0 16.5-7.2 32.1-19.6 43z"></path></svg></span></button><button class="feedbackButton_UgQs"><span role="img" aria-label="dislike" class="anticon anticon-dislike"><svg viewBox="64 64 896 896" focusable="false" data-icon="dislike" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M885.9 490.3c3.6-12 5.4-24.4 5.4-37 0-28.3-9.3-55.5-26.1-77.7 3.6-12 5.4-24.4 5.4-37 0-28.3-9.3-55.5-26.1-77.7 3.6-12 5.4-24.4 5.4-37 0-51.6-30.7-98.1-78.3-118.4a66.1 66.1 0 00-26.5-5.4H144c-17.7 0-32 14.3-32 32v364c0 17.7 14.3 32 32 32h129.3l85.8 310.8C372.9 889 418.9 924 470.9 924c29.7 0 57.4-11.8 77.9-33.4 20.5-21.5 31-49.7 29.5-79.4l-6-122.9h239.9c12.1 0 23.9-3.2 34.3-9.3 40.4-23.5 65.5-66.1 65.5-111 0-28.3-9.3-55.5-26.1-77.7zM184 456V172h81v284h-81zm627.2 160.4H496.8l9.6 198.4c.6 11.9-4.7 23.1-14.6 30.5-6.1 4.5-13.6 6.8-21.1 6.7a44.28 44.28 0 01-42.2-32.3L329 459.2V172h415.4a56.85 56.85 0 0133.6 51.8c0 9.7-2.3 18.9-6.9 27.3l-13.9 25.4 21.9 19a56.76 56.76 0 0119.6 43c0 9.7-2.3 18.9-6.9 27.3l-13.9 25.4 21.9 19a56.76 56.76 0 0119.6 43c0 9.7-2.3 18.9-6.9 27.3l-14 25.5 21.9 19a56.76 56.76 0 0119.6 43c0 19.1-11 37.5-28.8 48.4z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/metadata-integration/java/as-a-library"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Java Emitter</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/cli-commands/dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Dataset Command</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#using-pip" class="table-of-contents__link toc-highlight">Using pip</a></li></ul></li><li><a href="#starter-commands" class="table-of-contents__link toc-highlight">Starter Commands</a><ul><li><a href="#docker" class="table-of-contents__link toc-highlight">docker</a></li><li><a href="#ingest" class="table-of-contents__link toc-highlight">ingest</a><ul><li><a href="#ingest---dry-run" class="table-of-contents__link toc-highlight">ingest --dry-run</a></li><li><a href="#ingest-list-source-runs" class="table-of-contents__link toc-highlight">ingest list-source-runs</a></li><li><a href="#ingest---preview" class="table-of-contents__link toc-highlight">ingest --preview</a></li><li><a href="#ingest---no-default-report" class="table-of-contents__link toc-highlight">ingest --no-default-report</a></li></ul></li><li><a href="#ingest-deploy" class="table-of-contents__link toc-highlight">ingest deploy</a></li><li><a href="#init" class="table-of-contents__link toc-highlight">init</a><ul><li><a href="#environment-variables-supported" class="table-of-contents__link toc-highlight">Environment variables supported</a></li></ul></li><li><a href="#container" class="table-of-contents__link toc-highlight">container</a></li><li><a href="#check" class="table-of-contents__link toc-highlight">check</a><ul><li><a href="#restore-indices" class="table-of-contents__link toc-highlight">restore-indices</a></li><li><a href="#get-kafka-consumer-offsets" class="table-of-contents__link toc-highlight">get-kafka-consumer-offsets</a></li></ul></li><li><a href="#delete" class="table-of-contents__link toc-highlight">delete</a></li><li><a href="#exists" class="table-of-contents__link toc-highlight">exists</a></li><li><a href="#get" class="table-of-contents__link toc-highlight">get</a></li><li><a href="#put" class="table-of-contents__link toc-highlight">put</a><ul><li><a href="#put-aspect" class="table-of-contents__link toc-highlight">put aspect</a></li><li><a href="#put-platform" class="table-of-contents__link toc-highlight">put platform</a></li></ul></li><li><a href="#timeline" class="table-of-contents__link toc-highlight">timeline</a></li></ul></li><li><a href="#entity-specific-commands" class="table-of-contents__link toc-highlight">Entity Specific Commands</a><ul><li><a href="#dataset-dataset-entity" class="table-of-contents__link toc-highlight">dataset (Dataset Entity)</a></li><li><a href="#user-user-entity" class="table-of-contents__link toc-highlight">user (User Entity)</a></li><li><a href="#group-group-entity" class="table-of-contents__link toc-highlight">group (Group Entity)</a></li><li><a href="#dataproduct-data-product-entity" class="table-of-contents__link toc-highlight">dataproduct (Data Product Entity)</a><ul><li><a href="#upsert" class="table-of-contents__link toc-highlight">upsert</a></li><li><a href="#update" class="table-of-contents__link toc-highlight">update</a></li><li><a href="#diff" class="table-of-contents__link toc-highlight">diff</a></li><li><a href="#get-1" class="table-of-contents__link toc-highlight">get</a></li><li><a href="#add_asset" class="table-of-contents__link toc-highlight">add_asset</a></li><li><a href="#remove_asset" class="table-of-contents__link toc-highlight">remove_asset</a></li><li><a href="#add_owner" class="table-of-contents__link toc-highlight">add_owner</a></li><li><a href="#remove_owner" class="table-of-contents__link toc-highlight">remove_owner</a></li><li><a href="#set_description" class="table-of-contents__link toc-highlight">set_description</a></li><li><a href="#delete-1" class="table-of-contents__link toc-highlight">delete</a></li></ul></li></ul></li><li><a href="#miscellaneous-admin-commands" class="table-of-contents__link toc-highlight">Miscellaneous Admin Commands</a><ul><li><a href="#lite-experimental" class="table-of-contents__link toc-highlight">lite (experimental)</a></li><li><a href="#telemetry" class="table-of-contents__link toc-highlight">telemetry</a></li><li><a href="#migrate" class="table-of-contents__link toc-highlight">migrate</a><ul><li><a href="#dataplatform2instance" class="table-of-contents__link toc-highlight">dataplatform2instance</a></li></ul></li></ul></li><li><a href="#alternate-installation-options" class="table-of-contents__link toc-highlight">Alternate Installation Options</a><ul><li><a href="#using-docker" class="table-of-contents__link toc-highlight">Using docker</a></li><li><a href="#install-from-source" class="table-of-contents__link toc-highlight">Install from source</a></li></ul></li><li><a href="#installing-plugins" class="table-of-contents__link toc-highlight">Installing Plugins</a><ul><li><a href="#sources" class="table-of-contents__link toc-highlight">Sources</a></li><li><a href="#sinks" class="table-of-contents__link toc-highlight">Sinks</a></li><li><a href="#check-the-active-plugins" class="table-of-contents__link toc-highlight">Check the active plugins</a></li></ul></li><li><a href="#release-notes-and-cli-versions" class="table-of-contents__link toc-highlight">Release Notes and CLI versions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/docs/">Introduction</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/quickstart">Quickstart</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://datahub.com/slack" target="_blank" rel="noopener noreferrer" class="footer__link-item">Slack<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="footer__link-item">YouTube<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="footer__link-item">Blog<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/docs/townhalls">Town Halls</a></li><li class="footer__item"><a href="https://datahub.com/adoption-stories/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Adoption<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://demo.datahub.com/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Demo</a></li><li class="footer__item"><a href="https://feature-requests.datahubproject.io/roadmap" target="_blank" rel="noopener noreferrer" class="footer__link-item">Roadmap<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/docs/contributing">Contributing</a></li><li class="footer__item"><a href="https://github.com/datahub-project/datahub" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://feature-requests.datahubproject.io/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Feature Requests<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2015-2025 DataHub Project Authors.</div></div></div></footer></div>
|
||
<script src="/assets/js/runtime~main.ec69bdbc.js"></script>
|
||
<script src="/assets/js/main.89423a85.js"></script>
|
||
</body>
|
||
</html> |