mirror of
https://github.com/datahub-project/datahub.git
synced 2025-07-13 04:02:34 +00:00
140 lines
205 KiB
HTML
140 lines
205 KiB
HTML
|
<!doctype html>
|
|||
|
|
<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-docs/cli" data-has-hydrated="false">
|
|||
|
|
<head>
|
|||
|
|
<meta charset="UTF-8">
|
|||
|
|
<meta name="generator" content="Docusaurus v2.4.3">
|
|||
|
|
<title data-rh="true">DataHub CLI | DataHub</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://docs.datahub.com/docs/cli"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="DataHub CLI | DataHub"><meta data-rh="true" name="description" content="DataHub comes with a friendly cli called datahub that allows you to perform a lot of common operations using just the command line. DataHub maintains the pypi package for datahub."><meta data-rh="true" property="og:description" content="DataHub comes with a friendly cli called datahub that allows you to perform a lot of common operations using just the command line. DataHub maintains the pypi package for datahub."><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://docs.datahub.com/docs/cli"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/cli" hreflang="en"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/cli" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://RK0UG797F3-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/learn/rss.xml" title="DataHub RSS Feed">
|
|||
|
|
<link rel="alternate" type="application/atom+xml" href="/learn/atom.xml" title="DataHub Atom Feed">
|
|||
|
|
|
|||
|
|
<link rel="preconnect" href="https://www.google-analytics.com">
|
|||
|
|
<link rel="preconnect" href="https://www.googletagmanager.com">
|
|||
|
|
<script async src="https://www.googletagmanager.com/gtag/js?id=G-PKGVLETT4C"></script>
|
|||
|
|
<script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-PKGVLETT4C",{})</script>
|
|||
|
|
<link rel="preconnect" href="https://www.googletagmanager.com">
|
|||
|
|
<script>window.dataLayer=window.dataLayer||[]</script>
|
|||
|
|
<script>!function(e,t,a,n,g){e[n]=e[n]||[],e[n].push({"gtm.start":(new Date).getTime(),event:"gtm.js"});var m=t.getElementsByTagName(a)[0],r=t.createElement(a);r.async=!0,r.src="https://www.googletagmanager.com/gtm.js?id=GTM-5M8T9HNN",m.parentNode.insertBefore(r,m)}(window,document,"script","dataLayer")</script>
|
|||
|
|
|
|||
|
|
|
|||
|
|
<link rel="search" type="application/opensearchdescription+xml" title="DataHub" href="/opensearch.xml">
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
<meta httpequiv="Content-Security-Policy" content="frame-ancestors 'self' https://*.acryl.io https://acryldata.io http://localhost:*">
|
|||
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap">
|
|||
|
|
<script src="https://tools.luckyorange.com/core/lo.js?site-id=28ea8a38" async defer="defer"></script>
|
|||
|
|
<script src="/scripts/rb2b.js" async defer="defer"></script>
|
|||
|
|
<script src="https://app.revenuehero.io/scheduler.min.js"></script>
|
|||
|
|
<script src="https://tag.clearbitscripts.com/v1/pk_2e321cabe30432a5c44c0424781aa35f/tags.js" referrerpolicy="strict-origin-when-cross-origin"></script>
|
|||
|
|
<script src="/scripts/reo.js"></script>
|
|||
|
|
<script id="runllm-widget-script" type="module" src="https://widget.runllm.com" crossorigin="true" runllm-name="DataHub" runllm-assistant-id="81" runllm-position="BOTTOM_RIGHT" runllm-keyboard-shortcut="Mod+j" runllm-preset="docusaurus" runllm-theme-color="#1890FF" runllm-brand-logo="https://docs.datahub.com/img/datahub-logo-color-mark.svg" runllm-community-url="https://datahub.com/slack" runllm-community-type="slack" runllm-disable-ask-a-person="true" async></script><link rel="stylesheet" href="/assets/css/styles.d8fe2eb8.css">
|
|||
|
|
<link rel="preload" href="/assets/js/runtime~main.ec69bdbc.js" as="script">
|
|||
|
|
<link rel="preload" href="/assets/js/main.89423a85.js" as="script">
|
|||
|
|
</head>
|
|||
|
|
<body class="navigation-with-keyboard">
|
|||
|
|
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-5M8T9HNN" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
|||
|
|
|
|||
|
|
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
|
|||
|
|
<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:transparent;color:#ffffff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"><div class="shimmer-banner"><p>DataHub Secures $35 Million Series B</p><a href="https://datahub.com/news/series-b-announcement/" target="_blank" class="button"><div>Read the announcement<span> →</span></div></a></div></div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a href="https://datahub.com" target="_blank" rel="noopener noreferrer" class="navbar__brand"><div class="navbar__logo"><img src="/img/datahub-logo-color-light-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/datahub-logo-color-dark-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link versionNavItem_cbn8">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/cli">Next</a></li><li><a class="dropdown__link" href="/docs/1.1.0/cli">1.1.0</a></li><li><hr class="dropdown-separator" style="margin: 0.4rem;"></li><li><div class="dropdown__link"><b>Archived versions</b></div></li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/features">1.0.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/0.15.0/features">0.15.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-8jkm4uler-acryldata.vercel.app/docs/0.14.1/features">0.14.1
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-eue2qafvn-acryldata.vercel.app/docs/features">0.14.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-psat3nzgi-acryldata.vercel.app/docs/features">0.13.1
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-lzxh86531-acryldata.vercel.app/docs/features">0.13.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-2uuxmgza2-acryldata.vercel.app/docs/features">0.12.1
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-irpoe2osc-acryldata.vercel.app/docs/features">0.11.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-1gv2yzn9d-acryldata.vercel.app/docs/features">0.10.5
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li></ul></div></div><div class="navbar__items navbar__items--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/docs">Docs</a><a class="navbar__item navbar__link" href="/integrations">Integrations</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/weekly-demo" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Weekly Demo"></div><div class="title_c7DP">Weekly Demo</div></a></div><div><a href="https://datahub.com/use-cases" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-forum.png" alt="Use Cases"></div><div class="title_c7DP">Use Cases</div></a></div><div><a href="httpps://datahub.com/adoption-stories" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Adoption Stories"></div><div class="title_c7DP">Adoption Stories</div></a></div><div><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Blog"></div><div class="title_c7DP">Blog</div></a></div><div><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Youtube"></div><div class="title_c7DP">Youtube</div></a></div></div></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/slack/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Join Slack"></div><div class="title_c7DP">Join Slack</div></a></div><div><a href="https://datahub.com/events" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Events"></div><div class="title_c7DP">Events</div></a></div><div><a href="https://datahub.com/champions/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Champions"></div><div class="title_c7DP">Champions</div></a></div><div><a href="https://datahub.com/share-your-journey/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Share Your Journey"></div><div class="title_c7DP">Share Your Journey</div></a></div></div></ul></div><a href="https://datahub.com/products/why-datahub-cloud/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
|||
|
|
<style>
|
|||
|
|
.cloud-cta {
|
|||
|
|
color: var(--ifm-menu-color-active);
|
|||
|
|
font-weight: 600;
|
|||
|
|
background: linear-gradient(40deg, var(--ifm-menu-color-active), var(--ifm-menu-color-active));
|
|||
|
|
background-size: 200% 100%;
|
|||
|
|
-webkit-background-clip: text;
|
|||
|
|
background-clip: text;
|
|||
|
|
transition: background-image 0.3s ease;
|
|||
|
|
}
|
|||
|
|
.cloud-cta:hover {
|
|||
|
|
color: transparent;
|
|||
|
|
background: linear-gradient(40deg, var(--ifm-menu-color-active), #ff1493);
|
|||
|
|
background-size: 200% 100%;
|
|||
|
|
-webkit-background-clip: text;
|
|||
|
|
background-clip: text;
|
|||
|
|
animation: gradientShift 3s ease infinite;
|
|||
|
|
}
|
|||
|
|
@keyframes gradientShift {
|
|||
|
|
0%, 100% { background-position: 0% 50%; }
|
|||
|
|
50% { background-position: 100% 50%; }
|
|||
|
|
}
|
|||
|
|
</style>
|
|||
|
|
<div class="cloud-cta">Get Cloud</div>
|
|||
|
|
</a><a href="https://datahub.com/slack?utm_source=docs&utm_medium=header&utm_campaign=docs_header" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
|||
|
|
<style>
|
|||
|
|
.slack-logo:hover {
|
|||
|
|
opacity: 0.8;
|
|||
|
|
}
|
|||
|
|
</style>
|
|||
|
|
<img class="slack-logo" src="https://upload.wikimedia.org/wikipedia/commons/d/d5/Slack_icon_2019.svg" , alt="slack" , height="20px" style="margin: 10px 0 0 0;">
|
|||
|
|
</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Getting Started</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/features">What Is DataHub?</a><button aria-label="Toggle the collapsible sidebar category 'What Is DataHub?'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/features">Features</a><button aria-label="Toggle the collapsible sidebar category 'Features'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>DataHub Cloud</div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/managed-datahub-overview">DataHub Cloud Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/welcome-acryl">Getting Started with DataHub Cloud</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authentication/guides/sso/initialize-oidc">Configure Single Sign-On</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/remote-executor/about">Remote Executor</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/datahub-api/entity-events-api">DataHub API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--subli
|
|||
|
|
Like most command line tools, <code>--help</code> is your best friend. Use it to discover the capabilities of the cli and the different commands and sub-commands that are supported.</p><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub --help</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Usage: datahub [OPTIONS] COMMAND [ARGS]...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Options:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --debug / --no-debug Enable debug logging.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --log-file FILE Enable debug logging.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --debug-vars / --no-debug-vars Show variable values in stack traces. Implies --debug. While we try to avoid</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> printing sensitive information like passwords, this may still happen.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --version Show the version and exit.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> -dl, --detect-memory-leaks Run memory leak detection.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --help Show this message and exit.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Commands:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> actions <disabled due to missing dependencies></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> assertions A group of commands to interact with the Assertion entity in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> check Helper commands for checking various aspects of DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> container A group of commands to interact with containers in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> datacontract A group of commands to interact with the DataContract entity in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> dataproduct A group of commands to interact with the DataProduct entity in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> dataset A group of commands to interact with the Dataset entity in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> delete Delete metadata from DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> docker Helper commands for setting up and interacting with a local DataHub instance using Docker.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> exists A group of commands to check existence of entities in DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> forms A group of commands to inte
|
|||
|
|
We've ordered them roughly in the order we expect you to interact with these commands as you get deeper into the <code>datahub</code>-verse.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="docker">docker<a href="#docker" class="hash-link" aria-label="Direct link to docker" title="Direct link to docker"></a></h3><p>The <code>docker</code> command allows you to start up a local DataHub instance using <code>datahub docker quickstart</code>. You can also check if the docker cluster is healthy using <code>datahub docker check</code>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ingest">ingest<a href="#ingest" class="hash-link" aria-label="Direct link to ingest" title="Direct link to ingest"></a></h3><p>The <code>ingest</code> command allows you to ingest metadata from your sources using ingestion configuration files, which we call recipes.
|
|||
|
|
Source specific crawlers are provided by plugins and might sometimes need additional extras to be installed. See <a href="#installing-plugins">installing plugins</a> for more information.
|
|||
|
|
<a href="/docs/how/delete-metadata">Removing Metadata from DataHub</a> contains detailed instructions about how you can use the ingest command to perform operations like rolling-back previously ingested metadata through the <code>rollback</code> sub-command and listing all runs that happened through <code>list-runs</code> sub-command.</p><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">Usage: datahub [datahub-options] ingest [command-options]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Command Options:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> -c / --config Config file in .toml or .yaml format</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> -n / --dry-run Perform a dry run of the ingestion, essentially skipping writing to sink</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --preview Perform limited ingestion from the source to the sink to get a quick preview</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --preview-workunits The number of workunits to produce for preview</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --strict-warnings If enabled, ingestion runs with warnings will yield a non-zero error code</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --test-source-connection When set, ingestion will only test the source connection details from the recipe</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> --no-progress If enabled, mute intermediate progress ingestion reports</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ingest---dry-run">ingest --dry-run<a href="#ingest---dry-run" class="hash-link" aria-label="Direct link to ingest --dry-run" title="Direct link to ingest --dry-run"></a></h4><p>The <code>--dry-run</code> option of the <code>ingest</code> command performs all of the ingestion steps, except writing to the sink. This is useful to validate that the
|
|||
|
|
ingestion recipe is producing the desired metadata events before ingesting them into datahub.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Dry run</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml --dry-run</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Short-form</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml -n</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ingest-list-source-runs">ingest list-source-runs<a href="#ingest-list-source-runs" class="hash-link" aria-label="Direct link to ingest list-source-runs" title="Direct link to ingest list-source-runs"></a></h4><p>The <code>list-source-runs</code> option of the <code>ingest</code> command lists the previous runs, displaying their run ID, source name,
|
|||
|
|
start time, status, and source URN. This command allows you to filter results using the --urn option for URN-based
|
|||
|
|
filtering or the --source option to filter by source name (partial or complete matches are supported).</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># List all ingestion runs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest list-source-runs</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Filter runs by a source name containing "demo"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest list-source-runs --source </span><span class="token string" style="color:rgb(195, 232, 141)">"demo"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ingest---preview">ingest --preview<a href="#ingest---preview" class="hash-link" aria-label="Direct link to ingest --preview" title="Direct link to ingest --preview"></a></h4><p>The <code>--preview</code> option of the <code>ingest</code> command performs all of the ingestion steps, but limits the processing to only the first 10 workunits produced by the source.
|
|||
|
|
This option helps with quick end-to-end smoke testing of the ingestion recipe.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Preview</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml --preview</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Preview with dry-run</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml -n --preview</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>By default <code>--preview</code> creates 10 workunits. But if you wish to try producing more workunits you can use another option <code>--preview-workunits</code></p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Preview 20 workunits without sending anything to sink</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml -n --preview --preview-workunits</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">20</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ingest---no-default-report">ingest --no-default-report<a href="#ingest---no-default-report" class="hash-link" aria-label="Direct link to ingest --no-default-report" title="Direct link to ingest --no-default-report"></a></h4><p>By default, the cli sends an ingestion report to DataHub, which allows you to see the result of all cli-based ingestion in the UI. This can be turned off with the <code>--no-default-report</code> flag.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeB
|
|||
|
|
This command can also be used to schedule the ingestion while uploading or even to update existing sources. It will upload to the remote instance the
|
|||
|
|
CLI is connected to, not the sink of the recipe. Use <code>datahub init</code> to set the remote if not already set.</p><p>This command will automatically create a new recipe if it doesn't exist, or update it if it does.
|
|||
|
|
Note that this is a complete update, and will remove any options that were previously set.
|
|||
|
|
I.e: Not specifying a schedule in the cli update command will remove the schedule from the recipe to be updated.</p><p><strong>Basic example</strong></p><p>To schedule a recipe called "Snowflake Integration", to run at 5am every day, London time with the recipe configured in a local <code>recipe.yaml</code> file:</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest deploy --name </span><span class="token string" style="color:rgb(195, 232, 141)">"Snowflake Integration"</span><span class="token plain"> --schedule </span><span class="token string" style="color:rgb(195, 232, 141)">"5 * * * *"</span><span class="token plain"> --time-zone </span><span class="token string" style="color:rgb(195, 232, 141)">"Europe/London"</span><span class="token plain"> -c recipe.yaml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>By default, the ingestion recipe's identifier is generated by hashing the name.
|
|||
|
|
You can override the urn generation by passing the <code>--urn</code> flag to the CLI.</p><p><strong>Using <code>deployment</code> to avoid CLI args</strong></p><p>As an alternative to configuring settings from the CLI, all of these settings can also be set in the <code>deployment</code> field of the recipe.</p><div class="language-yml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># deployment_recipe.yml</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">deployment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"Snowflake Integration"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">schedule</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"5 * * * *"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">time_zone</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"Europe/London"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">source</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">...</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest deploy -c deployment_recipe.yml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor
|
|||
|
|
Running <code>datahub init</code> will allow you to customize the datahub instance you are communicating with. It has an optional <code>--use-password</code> option which allows to initialise the config using username, password. We foresee this mainly being used by admins as majority of organisations will be using SSO and there won't be any passwords to use.</p><p><strong><em>Note</em></strong>: Provide your GMS instance's host when the prompt asks you for the DataHub host.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain"># locally hosted example</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub init</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">/Users/user/.datahubenv already exists. Overwrite? [y/N]: y</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Configure which datahub instance to connect to</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Enter your DataHub host [http://localhost:8080]: http://localhost:8080</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Enter your DataHub access token []:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># acryl example</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub init</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">/Users/user/.datahubenv already exists. Overwrite? [y/N]: y</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Configure which datahub instance to connect to</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Enter your DataHub host [http://localhost:8080]: https://<your-instance-id>.acryl.io/gms</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Enter your DataHub access token []: <token generated from https://<your-instance-id>.acryl.io/settings/tokens></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="environment-variables-supported">Environment variables supported<a href="#environment-variables-supported" class="hash-link" aria-label="Direct link to Environment variables supported" title="Direct link to Environment variables supported"></a></h4><p>The environment variables listed below take precedence over the DataHub CLI config created through the <code>init</code> command.</p><ul><li><code>DATAHUB_SKIP_CONFIG</code> (default <code>false</code>) - Set to <code>true</code> to skip creating the configuration file.</li><li><code>DATAHUB_GMS_URL</code> (default <code>http://localhost:8080</code>) - Set to a URL of GMS instance</li><li><code>DATAHUB_GMS_HOST</code> (default <code>localhost</code>) - Set to a host of GMS instance. Prefer using <code>DATAHUB_GMS_URL</code> to set the URL.</li><li><code>DATAHUB_GMS_PORT</code> (default <code>8080</code>) - Set to a por
|
|||
|
|
The <code>check</code> command allows you to check if all plugins are loaded correctly as well as validate an individual MCE-file.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="restore-indices">restore-indices<a href="#restore-indices" class="hash-link" aria-label="Direct link to restore-indices" title="Direct link to restore-indices"></a></h4><p>This command allows you to restore indices for one or more <code>urn</code>.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub --debug check restore-indices --urn </span><span class="token string" style="color:rgb(195, 232, 141)">"URN"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>It can also take <code>--file</code> argument that points to a file that has list of urns like</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub check restore-indices --file ./urn.txt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>where urn.txt is like this</p><div class="language-urn.txt codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-urn.txt codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.schema1.test_all_nulls,PROD)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">urn:li:dataset:(urn:li:dataPlatform:platform1,test_db.schema2.test_complex_types,PROD)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">urn:li:dataset:(urn:li:dataPlatform:redshift,test_db.schema3.test_few_rows,PROD)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div
|
|||
|
|
For example the following command gets the ownership aspect from the dataset <code>urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)</code></p><div class="language-shell-session codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell-session codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token command shell-symbol important">$</span><span class="token command"> </span><span class="token command bash language-bash">datahub get --urn </span><span class="token command bash language-bash string" style="color:rgb(195, 232, 141)">"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)"</span><span class="token command bash language-bash"> --aspect ownership</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token output">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "ownership": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "lastModified": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "actor": "urn:li:corpuser:jdoe",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "time": 1680210917580</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "owners": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "owner": "urn:li:corpuser:jdoe",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "source": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "type": "SERVICE"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> "type": "NONE"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token output">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="put">put<a href="#put" class="hash-link" aria-label="Direct link to put" title="Direct link to put"></a></h3><p>The <code>put</code> group of commands allows you to write metadata into DataHub. This is a flexible way for you to issue edits to metadata from the command line.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="put-aspect">put aspect<a href="#put-aspect" class="hash-link" aria-label="Direct link to put aspect" title="Direct link to put aspect"></a></h4><p>The <strong>pu
|
|||
|
|
For example, the command shown below sets the <code>ownership</code> aspect of the dataset <code>urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)</code> to the value in the file <code>ownership.json</code>.
|
|||
|
|
The JSON in the <code>ownership.json</code> file needs to conform to the <a href="https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl" target="_blank" rel="noopener noreferrer"><code>Ownership</code></a> Aspect model as shown below.</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"owners"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"owner"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:corpuser:jdoe"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"DEVELOPER"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"owner"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:corpuser:jdub"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"DATAOWNER"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy
|
|||
|
|
the following command will show you the modifications to tags for a dataset for the past week. The output includes a computed semantic version,
|
|||
|
|
relevant for schema changes only currently, the target of the modification, and a description of the change including a timestamp.
|
|||
|
|
The default output is sanitized to be more readable, but the full API output can be obtained by passing the <code>--verbose</code> flag and
|
|||
|
|
to get the raw JSON difference in addition to the API output you can add the <code>--raw</code> flag. For more details about the feature please see <a href="/docs/dev-guides/timeline">the main feature page</a></p><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub timeline --urn "urn:li:dataset:(urn:li:dataPlatform:mysql,User.UserAccount,PROD)" --category TAG --start 7daysago</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">2022-02-17 14:03:42 - 0.0.0-computed</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> MODIFY TAG dataset:mysql:User.UserAccount : A change in aspect editableSchemaMetadata happened at time 2022-02-17 20:03:42.0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">2022-02-17 14:17:30 - 0.0.0-computed</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> MODIFY TAG dataset:mysql:User.UserAccount : A change in aspect editableSchemaMetadata happened at time 2022-02-17 20:17:30.118</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="entity-specific-commands">Entity Specific Commands<a href="#entity-specific-commands" class="hash-link" aria-label="Direct link to Entity Specific Commands" title="Direct link to Entity Specific Commands"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dataset-dataset-entity">dataset (Dataset Entity)<a href="#dataset-dataset-entity" class="hash-link" aria-label="Direct link to dataset (Dataset Entity)" title="Direct link to dataset (Dataset Entity)"></a></h3><p>The <code>dataset</code> command allows you to interact with Dataset entities in DataHub, including creating, updating, retrieving, and validating Dataset metadata.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Get a dataset and write to YAML file</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub dataset get --urn </span><span class="token string" style="color:rgb(195, 232, 141)">"urn:li:dataset:(urn:li:dataPlatform:hive,example_table,PROD)"</span><span class="token plain"> --to-file dataset.yaml</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Create or update dataset from YAML file</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub dataset upsert -f dataset.yaml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-labe
|
|||
|
|
It currently supports the <code>upsert</code> operation, which can be used to create a new user or update an existing one.
|
|||
|
|
For detailed information, please refer to <a href="/docs/api/tutorials/owners#upsert-users">Creating Users and Groups with Datahub CLI</a>.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub user upsert -f users.yaml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>An example of <code>users.yaml</code> would look like as in <a href="https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/cli_usage/user/bar.user.dhub.yaml" target="_blank" rel="noopener noreferrer">bar.user.dhub.yaml</a> file for the complete code.</p><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">id</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> bar@acryl.io</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">first_name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> The</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">last_name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> Bar</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">email</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> bar@acryl.io</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">slack</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"@the_bar_raiser"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">description</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"I like raising the bar higher"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">groups</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> foogroup@acryl.io</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></spa
|
|||
|
|
It currently supports the <code>upsert</code> operation, which can be used to create a new group or update an existing one with embedded Users.
|
|||
|
|
For more information, please refer to <a href="/docs/api/tutorials/owners#upsert-users">Creating Users and Groups with Datahub CLI</a>.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub group upsert -f group.yaml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>An example of <code>group.yaml</code> would look like as in <a href="https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/cli_usage/group/foo.group.dhub.yaml" target="_blank" rel="noopener noreferrer">foo.group.dhub.yaml</a> file for the complete code.</p><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token key atrule">id</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> foogroup@acryl.io</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">display_name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> Foo Group</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">admins</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> datahub</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token key atrule">members</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> bar@acryl.io </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># refer to a user either by id or by urn</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token key atrule">id</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> joe@acryl.io </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># inline specification of user</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">slack</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"@joe_shmoe"</span><span c
|
|||
|
|
See the <a href="/docs/dataproducts">Data Products</a> page for more details on what a Data Product is and how DataHub represents it.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub dataproduct --help</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Commands:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> upsert* Upsert attributes to a Data Product </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> DataHub</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> update Create or Update a Data Product </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> add_asset Add an asset to a Data Product</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> add_owner Add an owner to a Data Product</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> delete Delete a Data Product </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> DataHub.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token function" style="color:rgb(130, 170, 255)">diff</span><span class="token plain"> Diff a Data Product </span><span class="token function" style="color:rgb(130, 170, 255)">file</span><span class="token plain"> with its twin </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> DataHub</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> get Get a Data Product from DataHub</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> remove_asset Add an asset to a Data Product</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> remove_owner Remove an owner from a Data Product</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> set_description Set description </span><span class="token keyword" style="font-style:italic">for</span><span class="token plain"> a Data Product </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> DataHub</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Here we detail the sub-commands available under the dataproduct group of commands:</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="upsert">upsert<a href="#upsert" class="hash-link" aria-label="Direct link to upsert" title="Direct link to upsert"></a></h4><p>Use this to upsert a data product yaml file into DataHub. This will create the data product if it doesn't exist already. Remember, this will upsert all the fields that are specified in the yaml file and will not touch the fields that are not specified. For example, if you do not specify the <code>description</code> field in the yaml file, then
|
|||
|
|
We do not collect private information such as IP addresses, contents of ingestions, or credentials.
|
|||
|
|
The code responsible for collecting and broadcasting these events is open-source and can be found <a href="https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/telemetry/telemetry.py" target="_blank" rel="noopener noreferrer">within our GitHub</a>.</p><p>Telemetry is enabled by default, and the <code>telemetry</code> command lets you toggle the sending of these statistics via <code>telemetry enable/disable</code>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="migrate">migrate<a href="#migrate" class="hash-link" aria-label="Direct link to migrate" title="Direct link to migrate"></a></h3><p>The <code>migrate</code> group of commands allows you to perform certain kinds of migrations.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="dataplatform2instance">dataplatform2instance<a href="#dataplatform2instance" class="hash-link" aria-label="Direct link to dataplatform2instance" title="Direct link to dataplatform2instance"></a></h4><p>The <code>dataplatform2instance</code> migration command allows you to migrate your entities from an instance-agnostic platform identifier to an instance-specific platform identifier. If you have ingested metadata in the past for this platform and would like to transfer any important metadata over to the new instance-specific entities, then you should use this command. For example, if your users have added documentation or added tags or terms to your datasets, then you should run this command to transfer this metadata over to the new entities. For further context, read the Platform Instance Guide <a href="/docs/platform-instances">here</a>.</p><p>A few important options worth calling out:</p><ul><li>--dry-run / -n : Use this to get a report for what will be migrated before running</li><li>--force / -F : Use this if you know what you are doing and do not want to get a confirmation prompt before migration is started</li><li>--keep : When enabled, will preserve the old entities and not delete them. Default behavior is to soft-delete old entities.</li><li>--hard : When enabled, will hard-delete the old entities.</li></ul><p><strong><em>Note</em></strong>: Timeseries aspects such as Usage Statistics and Dataset Profiles are not migrated over to the new entity instances, you will get new data points created when you re-run ingestion using the <code>usage</code> or sources with profiling turned on.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="dry-run">Dry Run<a href="#dry-run" class="hash-link" aria-label="Direct link to Dry Run" title="Direct link to Dry Run"></a></h5><div class="language-console codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-console codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub migrate dataplatform2instance --platform elasticsearch --instance prod_index --dry-run</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Starting migration: platform:elasticsearch, instance=prod_index, force=False, dry-run=True</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">100% (25 of 25) |####################################################################################################################################################################################| Elapsed Time: 0:00:00 Time: 0:00:00</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[Dry Run] Migration Report:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--------------</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[Dry Run] Migration Run Id: migrate-5710349c-1ec7-4b83-a7d3-47d71b7e972e</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">[Dry Run] Num entities created = 25</span><br></span><span clas
|
|||
|
|
We have prebuilt images available on <a href="https://hub.docker.com/r/acryldata/datahub-ingestion" target="_blank" rel="noopener noreferrer">Docker hub</a>. All plugins will be installed and enabled automatically.</p><p>You can use the <code>datahub-ingestion</code> docker image as explained in <a href="/docs/docker">Docker Images</a>. In case you are using Kubernetes you can start a pod with the <code>datahub-ingestion</code> docker image, log onto a shell on the pod and you should have the access to datahub CLI in your kubernetes cluster.</p><p><em>Limitation: the datahub_docker.sh convenience script assumes that the recipe and any input/output files are accessible in the current working directory or its subdirectories. Files outside the current working directory will not be found, and you'll need to invoke the Docker image directly.</em></p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># Assumes the DataHub repo is cloned locally.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">./metadata-ingestion/scripts/datahub_docker.sh ingest -c ./examples/recipes/example_to_datahub_rest.yml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-from-source">Install from source<a href="#install-from-source" class="hash-link" aria-label="Direct link to Install from source" title="Direct link to Install from source"></a></h3><p>If you'd like to install from source, see the <a href="/docs/metadata-ingestion/developing">developer guide</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="installing-plugins">Installing Plugins<a href="#installing-plugins" class="hash-link" aria-label="Direct link to Installing Plugins" title="Direct link to Installing Plugins"></a></h2><p>We use a plugin architecture so that you can install only the dependencies you actually need. Click the plugin name to learn more about the specific source recipe and any FAQs!</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sources">Sources<a href="#sources" class="hash-link" aria-label="Direct link to Sources" title="Direct link to Sources"></a></h3><p>Please see our <a href="/integrations">Integrations page</a> if you want to filter on the features offered by each source.</p><table><thead><tr><th>Plugin Name</th><th>Install Command</th><th>Provides</th></tr></thead><tbody><tr><td><a href="/docs/generated/ingestion/sources/metadata-file">metadata-file</a></td><td><em>included by default</em></td><td>File source and sink</td></tr><tr><td><a href="/docs/generated/ingestion/sources/athena">athena</a></td><td><code>pip install 'acryl-datahub[athena]'</code></td><td>AWS Athena source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/bigquery">bigquery</a></td><td><code>pip install 'acryl-datahub[bigquery]'</code></td><td>BigQuery source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/file-based-lineage">datahub-lineage-file</a></td><td><em>no additional dependencies</em></td><td>Lineage File source</td></tr><tr><td><a href="/docs/generated/ingestion/sources/bu
|
|||
|
|
<script src="/assets/js/runtime~main.ec69bdbc.js"></script>
|
|||
|
|
<script src="/assets/js/main.89423a85.js"></script>
|
|||
|
|
</body>
|
|||
|
|
</html>
|