mirror of
https://github.com/datahub-project/datahub.git
synced 2025-07-25 10:30:04 +00:00
165 lines
173 KiB
HTML
165 lines
173 KiB
HTML
|
<!doctype html>
|
|||
|
|
<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-docs/how/updating-datahub" data-has-hydrated="false">
|
|||
|
|
<head>
|
|||
|
|
<meta charset="UTF-8">
|
|||
|
|
<meta name="generator" content="Docusaurus v2.4.3">
|
|||
|
|
<title data-rh="true">Updating DataHub | DataHub</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://docs.datahub.com/docs/how/updating-datahub"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Updating DataHub | DataHub"><meta data-rh="true" name="description" content="<!--"><meta data-rh="true" property="og:description" content="<!--"><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://docs.datahub.com/docs/how/updating-datahub"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/how/updating-datahub" hreflang="en"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/how/updating-datahub" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://RK0UG797F3-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/learn/rss.xml" title="DataHub RSS Feed">
|
|||
|
|
<link rel="alternate" type="application/atom+xml" href="/learn/atom.xml" title="DataHub Atom Feed">
|
|||
|
|
|
|||
|
|
<link rel="preconnect" href="https://www.google-analytics.com">
|
|||
|
|
<link rel="preconnect" href="https://www.googletagmanager.com">
|
|||
|
|
<script async src="https://www.googletagmanager.com/gtag/js?id=G-PKGVLETT4C"></script>
|
|||
|
|
<script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-PKGVLETT4C",{})</script>
|
|||
|
|
<link rel="preconnect" href="https://www.googletagmanager.com">
|
|||
|
|
<script>window.dataLayer=window.dataLayer||[]</script>
|
|||
|
|
<script>!function(e,t,a,n,g){e[n]=e[n]||[],e[n].push({"gtm.start":(new Date).getTime(),event:"gtm.js"});var m=t.getElementsByTagName(a)[0],r=t.createElement(a);r.async=!0,r.src="https://www.googletagmanager.com/gtm.js?id=GTM-5M8T9HNN",m.parentNode.insertBefore(r,m)}(window,document,"script","dataLayer")</script>
|
|||
|
|
|
|||
|
|
|
|||
|
|
<link rel="search" type="application/opensearchdescription+xml" title="DataHub" href="/opensearch.xml">
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
<meta httpequiv="Content-Security-Policy" content="frame-ancestors 'self' https://*.acryl.io https://acryldata.io http://localhost:*">
|
|||
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap">
|
|||
|
|
<script src="https://tools.luckyorange.com/core/lo.js?site-id=28ea8a38" async defer="defer"></script>
|
|||
|
|
<script src="/scripts/rb2b.js" async defer="defer"></script>
|
|||
|
|
<script src="https://app.revenuehero.io/scheduler.min.js"></script>
|
|||
|
|
<script src="https://tag.clearbitscripts.com/v1/pk_2e321cabe30432a5c44c0424781aa35f/tags.js" referrerpolicy="strict-origin-when-cross-origin"></script>
|
|||
|
|
<script src="/scripts/reo.js"></script>
|
|||
|
|
<script id="runllm-widget-script" type="module" src="https://widget.runllm.com" crossorigin="true" runllm-name="DataHub" runllm-assistant-id="81" runllm-position="BOTTOM_RIGHT" runllm-keyboard-shortcut="Mod+j" runllm-preset="docusaurus" runllm-theme-color="#1890FF" runllm-brand-logo="https://docs.datahub.com/img/datahub-logo-color-mark.svg" runllm-community-url="https://datahub.com/slack" runllm-community-type="slack" runllm-disable-ask-a-person="true" async></script><link rel="stylesheet" href="/assets/css/styles.d8fe2eb8.css">
|
|||
|
|
<link rel="preload" href="/assets/js/runtime~main.310f59c4.js" as="script">
|
|||
|
|
<link rel="preload" href="/assets/js/main.49198d73.js" as="script">
|
|||
|
|
</head>
|
|||
|
|
<body class="navigation-with-keyboard">
|
|||
|
|
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-5M8T9HNN" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
|||
|
|
|
|||
|
|
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
|
|||
|
|
<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:transparent;color:#ffffff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"><div class="shimmer-banner"><p>DataHub Secures $35 Million Series B</p><a href="https://datahub.com/news/series-b-announcement/" target="_blank" class="button"><div>Read the announcement<span> →</span></div></a></div></div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a href="https://datahub.com" target="_blank" rel="noopener noreferrer" class="navbar__brand"><div class="navbar__logo"><img src="/img/datahub-logo-color-light-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/datahub-logo-color-dark-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link versionNavItem_cbn8">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/how/updating-datahub">Next</a></li><li><a class="dropdown__link" href="/docs/1.1.0/how/updating-datahub">1.1.0</a></li><li><hr class="dropdown-separator" style="margin: 0.4rem;"></li><li><div class="dropdown__link"><b>Archived versions</b></div></li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/features">1.0.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/0.15.0/features">0.15.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-8jkm4uler-acryldata.vercel.app/docs/0.14.1/features">0.14.1
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-eue2qafvn-acryldata.vercel.app/docs/features">0.14.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-psat3nzgi-acryldata.vercel.app/docs/features">0.13.1
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-lzxh86531-acryldata.vercel.app/docs/features">0.13.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-2uuxmgza2-acryldata.vercel.app/docs/features">0.12.1
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-irpoe2osc-acryldata.vercel.app/docs/features">0.11.0
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li><li>
|
|||
|
|
<a class="dropdown__link" href="https://docs-website-1gv2yzn9d-acryldata.vercel.app/docs/features">0.10.5
|
|||
|
|
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
|||
|
|
</a>
|
|||
|
|
</li></ul></div></div><div class="navbar__items navbar__items--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/docs">Docs</a><a class="navbar__item navbar__link" href="/integrations">Integrations</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/weekly-demo" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Weekly Demo"></div><div class="title_c7DP">Weekly Demo</div></a></div><div><a href="https://datahub.com/use-cases" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-forum.png" alt="Use Cases"></div><div class="title_c7DP">Use Cases</div></a></div><div><a href="httpps://datahub.com/adoption-stories" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Adoption Stories"></div><div class="title_c7DP">Adoption Stories</div></a></div><div><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Blog"></div><div class="title_c7DP">Blog</div></a></div><div><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Youtube"></div><div class="title_c7DP">Youtube</div></a></div></div></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/slack/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Join Slack"></div><div class="title_c7DP">Join Slack</div></a></div><div><a href="https://datahub.com/events" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Events"></div><div class="title_c7DP">Events</div></a></div><div><a href="https://datahub.com/champions/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Champions"></div><div class="title_c7DP">Champions</div></a></div><div><a href="https://datahub.com/share-your-journey/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Share Your Journey"></div><div class="title_c7DP">Share Your Journey</div></a></div></div></ul></div><a href="https://datahub.com/products/why-datahub-cloud/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
|||
|
|
<style>
|
|||
|
|
.cloud-cta {
|
|||
|
|
color: var(--ifm-menu-color-active);
|
|||
|
|
font-weight: 600;
|
|||
|
|
background: linear-gradient(40deg, var(--ifm-menu-color-active), var(--ifm-menu-color-active));
|
|||
|
|
background-size: 200% 100%;
|
|||
|
|
-webkit-background-clip: text;
|
|||
|
|
background-clip: text;
|
|||
|
|
transition: background-image 0.3s ease;
|
|||
|
|
}
|
|||
|
|
.cloud-cta:hover {
|
|||
|
|
color: transparent;
|
|||
|
|
background: linear-gradient(40deg, var(--ifm-menu-color-active), #ff1493);
|
|||
|
|
background-size: 200% 100%;
|
|||
|
|
-webkit-background-clip: text;
|
|||
|
|
background-clip: text;
|
|||
|
|
animation: gradientShift 3s ease infinite;
|
|||
|
|
}
|
|||
|
|
@keyframes gradientShift {
|
|||
|
|
0%, 100% { background-position: 0% 50%; }
|
|||
|
|
50% { background-position: 100% 50%; }
|
|||
|
|
}
|
|||
|
|
</style>
|
|||
|
|
<div class="cloud-cta">Get Cloud</div>
|
|||
|
|
</a><a href="https://datahub.com/slack?utm_source=docs&utm_medium=header&utm_campaign=docs_header" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
|||
|
|
<style>
|
|||
|
|
.slack-logo:hover {
|
|||
|
|
opacity: 0.8;
|
|||
|
|
}
|
|||
|
|
</style>
|
|||
|
|
<img class="slack-logo" src="https://upload.wikimedia.org/wikipedia/commons/d/d5/Slack_icon_2019.svg" , alt="slack" , height="20px" style="margin: 10px 0 0 0;">
|
|||
|
|
</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Getting Started</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/features">What Is DataHub?</a><button aria-label="Toggle the collapsible sidebar category 'What Is DataHub?'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/features">Features</a><button aria-label="Toggle the collapsible sidebar category 'Features'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>DataHub Cloud</div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/managed-datahub-overview">DataHub Cloud Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/welcome-acryl">Getting Started with DataHub Cloud</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authentication/guides/sso/initialize-oidc">Configure Single Sign-On</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/remote-executor/about">Remote Executor</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/datahub-api/entity-events-api">DataHub API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--subli
|
|||
|
|
(after 10d) or are timeseries <em>entities</em> (dataprocess, execution requests)
|
|||
|
|
will be removed automatically using logic in the <code>datahub-gc</code> ingestion
|
|||
|
|
source.</li><li>#12067 - Default behavior of DataJobPatchBuilder in Python sdk has been
|
|||
|
|
changed to NOT fill out <code>created</code> and <code>lastModified</code> auditstamps by default
|
|||
|
|
for input and output dataset edges. This should not have any user-observable
|
|||
|
|
impact (time-based lineage viz will still continue working based on observed time), but could break assumptions previously being made by clients.</li><li>#12158 - Users provisioned with <code>user.props</code> will need to be enabled before login in order to be granted access to DataHub.</li><li>#13273 - When using Opensearch, we will use 'zstd-no-dict' codec instead of 'default'. Elasticsearch still uses 'default'</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-3">Potential Downtime<a href="#potential-downtime-3" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-3">Deprecations<a href="#deprecations-3" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><ul><li><p>#12056: The DataHub Airflow plugin no longer supports Airflow 2.1 and Airflow 2.2.</p></li><li><p>#11701: The Fivetran <code>sources_to_database</code> field is deprecated in favor of setting directly within <code>sources_to_platform_instance.<key>.database</code>.</p></li><li><p>#11560 - The PowerBI ingestion source configuration option include_workspace_name_in_dataset_urn determines whether the workspace name is included in the PowerBI dataset's URN.<br> PowerBI allows to have identical name of semantic model and their tables across the workspace, It will overwrite the semantic model in-case of multi-workspace ingestion.<br>
|
|||
|
|
Entity urn with <code>include_workspace_name_in_dataset_urn: false</code></p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain"> urn:li:dataset:(urn:li:dataPlatform:powerbi,[<PlatformInstance>.]<SemanticModelName>.<TableName>,<ENV>)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Entity urn with <code>include_workspace_name_in_dataset_urn: true</code></p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain"> urn:li:dataset:(urn:li:dataPlatform:powerbi,[<PlatformInstance>.].<WorkspaceName>.<SemanticModelName>.<TableName>,<ENV>)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The config <code>include_workspace_name_in_dataset_urn</code> is default to <code>false</code> for backward compatibility, However, we recommend enabling this flag after performing the necessary cleanup.
|
|||
|
|
If stateful ingestion is enabled, running ingestion with the latest CLI version will handle the cleanup automatically. Otherwise, we recommend soft deleting all powerbi data via the DataHub CLI:
|
|||
|
|
<code>datahub delete --platform powerbi --soft</code> and then re-ingest with the latest CLI version, ensuring the <code>include_workspace_name_in_dataset_urn</code> configuration is set to true.</p></li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-3">Other Notable Changes<a href="#other-notable-changes-3" class="hash-link" aria-label="Direct link to Other Notable Changes" title="Direct link to Other Notable Changes"></a></h3><ul><li>#12236: Data flow and data job entities may additionally produce container aspect that will require a corresponding upgrade of server. Otherwise server can reject the aspect.</li><li>#12056: The DataHub Airflow plugin now defaults to the v2 plugin implementation.</li><li>#11742: For PowerBi ingestion, <code>use_powerbi_email</code> is now enabled by default when extracting ownership information.</li><li>#11549 - Manage Operations Privilege is extended from throttle control to all system management and operations APIs.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="0141">0.14.1<a href="#0141" class="hash-link" aria-label="Direct link to 0.14.1" title="Direct link to 0.14.1"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-4">Breaking Changes<a href="#breaking-changes-4" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li><p>#9857 (#10773) <code>lower</code> method was removed from <code>get_db_name</code> of <code>SQLAlchemySource</code> class. This change will affect the urns of all related to <code>SQLAlchemySource</code> entities.</p><p>Old <code>urn</code>, where <code>data_base_name</code> is <code>Some_Database</code>:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">- urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>New <code>urn</code>, where <code>data_base_name</code> is <code>Some_Database</code>:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">- urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Re-running with stateful ingestion should automatically clear up the entities with old URNS and add entities with new URNs, therefore not duplicating the
|
|||
|
|
behavior is required.</li><li>#10814 Data flow info and data job info aspect will produce an additional field that will require a corresponding upgrade of server. Otherwise server can reject the aspects.</li><li>#10868 - OpenAPI V3 - Creation of aspects will need to be wrapped within a <code>value</code> key and the API is now symmetric with respect to input and outputs.</li></ul><p>Example Global Tags Aspect:</p><p>Previous:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"tags"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"tag"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"string"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"context"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"string"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>New (optional fields <code>systemMetadata</code> and <code>headers</code>):</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"value"</span><span class="token o
|
|||
|
|
will be deprecated in no less than 6 months. Endpoints will be replaced with equivalents in the <code>/v2</code> or <code>/v3</code> APIs.
|
|||
|
|
No loss of functionality expected unless explicitly mentioned in Breaking Changes.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-5">Other Notable Changes<a href="#other-notable-changes-5" class="hash-link" aria-label="Direct link to Other Notable Changes" title="Direct link to Other Notable Changes"></a></h3><ul><li>#10498 - Tableau ingestion can now be configured to ingest multiple sites at once and add the sites as containers. The feature is currently only available for Tableau Server.</li><li>#10466 - Extends configuration in <code>~/.datahubenv</code> to match <code>DatahubClientConfig</code> object definition. See full configuration in <a href="https://docs.datahub.com/docs/python-sdk/clients/" target="_blank" rel="noopener noreferrer">https://docs.datahub.com/docs/python-sdk/clients/</a>. The CLI should now respect the updated configurations specified in <code>~/.datahubenv</code> across its functions and utilities. This means that for systems where ssl certification is disabled, setting <code>disable_ssl_verification: true</code> in <code>~./datahubenv</code> will apply to all CLI calls.</li><li>#11002 - We will not auto-generate a <code>~/.datahubenv</code> file. You must either run <code>datahub init</code> to create that file, or set environment variables so that the config is loaded.</li><li>#11023 - Added a new parameter to datahub's <code>put</code> cli command: <code>--run-id</code>. This parameter is useful to associate a given write to an ingestion process. A use-case can be mimick transformers when a transformer for aspect being written does not exist.</li><li>#11051 - Ingestion reports will now trim the summary text to a maximum of 800k characters to avoid generating <code>dataHubExecutionRequestResult</code> that are too large for GMS to handle.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="0133">0.13.3<a href="#0133" class="hash-link" aria-label="Direct link to 0.13.3" title="Direct link to 0.13.3"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-6">Breaking Changes<a href="#breaking-changes-6" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li>#10419 - <code>aws_region</code> is now a required configuration in the DynamoDB connector. The connector will no longer loop through all AWS regions; instead, it will only use the region passed into the recipe configuration.</li><li>#10389 - Custom validators, mutators, side-effects dropped a previously required constructor</li><li>#10472 - <code>RVW</code> added as a FabricType. No rollbacks allowed once metadata with this fabric type is added without manual cleanups in databases.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-6">Potential Downtime<a href="#potential-downtime-6" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-6">Deprecations<a href="#deprecations-6" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-change">Other Notable Change<a href="#other-notable-change" class="hash-link" aria-label="Direct link to Other Notable Change" title="Direct link to Other Notable Change"></a></h3><h2 class="anchor anchorWithStickyNavbar_LWe7" id="0131">0.13.1<a href="#0131" class="hash-link" aria-label="Direct link to 0.13.1" title="Direct link to 0.13.1"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-7">Breaking Changes<a href="#breaking-changes-7" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li>#9934 and #10075 - Stateful ingestion is now enabled by default if a <code>pipeline_name</code> is set and either a datahub-rest sink or <code>datahub_api</code> is specified. It will still be disabled by default when an
|
|||
|
|
Entity Hierarchy with <code>include_metastore: true</code> (Old)</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">- UC Metastore</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> - Catalog</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> - Schema</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> - Table</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Entity Hierarchy with <code>include_metastore: false</code> (New)</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">- Catalog</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> - Schema</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> - Table</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We recommend using <code>platform_instance</code> for differentiating across metastores.</p><p>If stateful ingestion is enabled, running ingestion with latest cli version will perform all required cleanup. Otherwise, we recommend soft deleting all databricks data via the DataHub CLI:
|
|||
|
|
<code>datahub delete --platform databricks --soft</code> and then reingesting with latest cli version.</p></li><li><p>#9601 - The Unity Catalog(UC) ingestion source config <code>include_hive_metastore</code> is now enabled by default. This requires config <code>warehouse_id</code> to be set. You can disable <code>include_hive_metastore</code> by setting it to <code>False</code> to avoid ingesting legacy hive metastore catalog in Databricks.</p></li><li><p>#9904 - The default Redshift <code>table_lineage_mode</code> is now MIXED, instead of <code>STL_SCAN_BASED</code>. Improved lineage generation is also available by enabling <code>use_lineaege_v2</code>. This v2 implementation will become the default in a future release.</p></li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-8">Potential Downtime<a href="#potential-downtime-8" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-8">Deprecations<a href="#deprecations-8" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><ul><li>Spark 2.x (including previous JDK8 build requirements)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-7">Other Notable Changes<a href="#other-notable-changes-7" class="hash-link" aria-label="Direct link to Other Notable Changes" title="Direct link to Other Notable Changes"></a></h3><h2 class="anchor anchorWithStickyNavbar_LWe7" id="0121">0.12.1<a href="#0121" class="hash-link" aria-label="Direct link to 0.12.1" title="Direct link to 0.12.1"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-9">Breaking Changes<a href="#breaking-changes-9" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li>#9244: The <code>redshift-legacy</code> and <code>redshift-legacy-usage</code> sources, which have been deprecated for >6 months, have been removed. The new <code>redshift</code> source is a superset of the functionality provided by those legacy sources.</li><li><code>database_alias</code> config is no longer supported in SQL sources namely - Redshift, MySQL, Oracle, Postgres, Trino, Presto-on-hive. The config will automatically be ignored if it's present in your recipe. It has been deprecated since v0.9.6.</li><li>#9257: The Python SDK urn types are now autogenerated. The new classes are largely backwards compatible with the previous, manually written classes, but many older methods are now deprecated in favor of a more uniform interface. The only breaking change is that the signature for the director constructor e.g. <code>TagUrn("tag", ["tag_name"])</code> is no longer supported, and the simpler <code>TagUrn("tag_name")</code> should be used instead.
|
|||
|
|
The canonical place to import the urn classes from is <code>datahub.metadata.urns.*</code>. Other import paths, like <code>datahub.utilities.urns.corpuser_urn.CorpuserUrn</code> are retained for backwards compatibility, but are considered deprecated.</li><li>#9286: The <code>DataHubRestEmitter.emit</code> method no longer returns anything. It previously returned a tuple of timestamps.</li><li>#8951: A great expectations based profiler has been added for the Unity Catalog source.
|
|||
|
|
To use the old profiler, set <code>method: analyze</code> under the <code>profiling</code> section in your recipe.
|
|||
|
|
To use the new profiler, set <code>method: ge</code>. Profiling is disabled by default, so to enable it,
|
|||
|
|
one of these methods must be specified.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-9">Potential Downtime<a href="#potential-downtime-9" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-9">Deprecations<a href="#deprecations-9" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-8">Other Notable Changes<a href="#other-notable-changes-8" class="hash-link" aria-label="Direct link to Other Notable Changes" title="Direct link to Other Notable Changes"></a></h3><h2 class="anchor anchorWithStickyNavbar_LWe7" id="0120">0.12.0<a href="#0120" class="hash-link" aria-label="Direct link to 0.12.0" title="Direct link to 0.12.0"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-10">Breaking Changes<a href="#breaking-changes-10" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li>#8687 (datahub-helm #365 #353) - If Helm is used for installation and Neo4j is enabled, update the prerequisites Helm chart to version >=0.1.2 and adjust your value overrides in the <code>neo4j:</code> section according to the new structure.</li><li>#9044 - GraphQL APIs for adding ownership now expect either an <code>ownershipTypeUrn</code> referencing a customer ownership type or a (deprecated) <code>type</code>. Where before adding an ownership without a concrete type was allowed, this is no longer the case. For simplicity you can use the <code>type</code> parameter which will get translated to a custom ownership type internally if one exists for the type being added.</li><li>#9010 - In Redshift source's config <code>incremental_lineage</code> is set default to off.</li><li>#8810 - Removed support for SQLAlchemy 1.3.x. Only SQLAlchemy 1.4.x is supported now.</li><li>#8942 - Removed <code>urn:li:corpuser:datahub</code> owner for the <code>Measure</code>, <code>Dimension</code> and <code>Temporal</code> tags emitted
|
|||
|
|
by Looker and LookML source connectors.</li><li>#8853 - The Airflow plugin no longer supports Airflow 2.0.x or Python 3.7. See the docs for more details.</li><li>#8853 - Introduced the Airflow plugin v2. If you're using Airflow 2.3+, the v2 plugin will be enabled by default, and so you'll need to switch your requirements to include <code>pip install 'acryl-datahub-airflow-plugin[plugin-v2]'</code>. To continue using the v1 plugin, set the <code>DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN</code> environment variable to <code>true</code>.</li><li>#8943 - The Unity Catalog ingestion source has a new option <code>include_metastore</code>, which will cause all urns to be changed when disabled.
|
|||
|
|
This is currently enabled by default to preserve compatibility, but will be disabled by default and then removed in the future.
|
|||
|
|
If stateful ingestion is enabled, simply setting <code>include_metastore: false</code> will perform all required cleanup.
|
|||
|
|
Otherwise, we recommend soft deleting all databricks data via the DataHub CLI:
|
|||
|
|
<code>datahub delete --platform databricks --soft</code> and then reingesting with <code>include_metastore: false</code>.</li><li>#8846 - Changed enum values in resource filters used by policies. <code>RESOURCE_TYPE</code> became <code>TYPE</code> and <code>RESOURCE_URN</code> became <code>URN</code>.
|
|||
|
|
Any existing policies using these filters (i.e. defined for particular <code>urns</code> or <code>types</code> such as <code>dataset</code>) need to be upgraded
|
|||
|
|
manually, for example by retrieving their respective <code>dataHubPolicyInfo</code> aspect and changing part using filter i.e.</li></ul><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">"resources"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">"filter"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">"criteria"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">"field"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"RESOURCE_TYPE"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">"condition"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"EQUALS"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">"values"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"dataset"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-
|
|||
|
|
This means that any <code>dataset_pattern</code> or <code>schema_pattern</code> specified will be matched on the fully
|
|||
|
|
qualified dataset name, i.e. <code><project_name>.<dataset_name></code>. We attempt to support the old
|
|||
|
|
pattern format by prepending <code>.*\\.</code> to dataset patterns lacking a period, so in most cases this
|
|||
|
|
should not cause any issues. However, if you have a complex dataset pattern, we recommend you
|
|||
|
|
manually convert it to the fully qualified format to avoid any potential issues.</li><li>#9110 - The Unity Catalog source will now generate urns based on <code>env</code> properly. If you have
|
|||
|
|
been setting <code>env</code> in your recipe to something besides <code>PROD</code>, we will now generate urns
|
|||
|
|
with that new env variable, invalidating your existing urns.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-10">Potential Downtime<a href="#potential-downtime-10" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-10">Deprecations<a href="#deprecations-10" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-9">Other Notable Changes<a href="#other-notable-changes-9" class="hash-link" aria-label="Direct link to Other Notable Changes" title="Direct link to Other Notable Changes"></a></h3><ul><li>Session token configuration has changed, all previously created session tokens will be invalid and users will be prompted to log in. Expiration time has also been shortened which may result in more login prompts with the default settings.
|
|||
|
|
There should be no other interruption due to this change.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="0110">0.11.0<a href="#0110" class="hash-link" aria-label="Direct link to 0.11.0" title="Direct link to 0.11.0"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-11">Breaking Changes<a href="#breaking-changes-11" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-11">Potential Downtime<a href="#potential-downtime-11" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><ul><li>#8611 Search improvements requires reindexing indices. A <code>system-update</code> job will run which will set indices to read-only and create a backup/clone of each index. During the reindexing new components will be prevented from start-up until the reindex completes. The logs of this job will indicate a % complete per index. Depending on index sizes and infrastructure this process can take 5 minutes to hours however as a rough estimate 1 hour for every 2.3 million entities.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-11">Deprecations<a href="#deprecations-11" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><ul><li>#8525: In LDAP ingestor, the <code>manager_pagination_enabled</code> changed to general <code>pagination_enabled</code></li><li>MAE Events are no longer produced. MAE events have been deprecated for over a year.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-10">Other Notable Changes<a href="#other-notable-changes-10" class="hash-link" aria-label="Direct link to Other Notable Changes" title="Direct link to Other Notable Changes"></a></h3><ul><li>In this release we now enable you to create and delete pinned announcements on your DataHub homepage! If you have the “Manage Home Page Posts” platform privilege you’ll see a new section in settings called “Home Page Posts” where you can create and delete text posts and link posts that your users see on the home page.</li><li>The new search and browse experience, which was first made available in the previous release behind a feature flag, is now on by default. Check out our release notes for v0.10.5 to get more information and documentation on this new Browse experience.</li><li>In addition to the ranking changes mentioned above, this release includes changes to the highlighting of search entities to understand why they match your query. You can also sort your results alphabetically or by last updated times, in addition to relevance. In this release, we suggest a correction if your query has a typo in it.</li><li>#8300: Clickhouse source now inherited from TwoTierSQLAlchemy. In old way we have platform_instance -> container -> co
|
|||
|
|
container db (None) -> container schema and now we have platform_instance -> container database.</li><li>#8300: Added <code>uri_opts</code> argument; now we can add any options for clickhouse client.</li><li>#8659: BigQuery ingestion no longer creates DataPlatformInstance aspects by default.
|
|||
|
|
This will only affect users that were depending on this aspect for custom functionality,
|
|||
|
|
and can be enabled via the <code>include_data_platform_instance</code> config option.</li><li>OpenAPI entity and aspect endpoints expanded to improve developer experience when using this API with additional aspects to be added in the near future.</li><li>The CLI now supports recursive deletes.</li><li>Batching of default aspects on initial ingestion (SQL)</li><li>Improvements to multi-threading. Ingestion recipes, if previously reduced to 1 thread, can be restored to the 15 thread default.</li><li>Gradle 7 upgrade moderately improves build speed</li><li>DataHub Ingestion slim images reduced in size by 2GB+</li><li>Glue Schema Registry fixed</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="0105">0.10.5<a href="#0105" class="hash-link" aria-label="Direct link to 0.10.5" title="Direct link to 0.10.5"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-12">Breaking Changes<a href="#breaking-changes-12" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li>#8201: Python SDK: In the DataFlow class, the <code>cluster</code> argument is deprecated in favor of <code>env</code>.</li><li>#8263: Okta source config option <code>okta_profile_to_username_attr</code> default changed from <code>login</code> to <code>email</code>.
|
|||
|
|
This determines which Okta profile attribute is used for the corresponding DataHub user
|
|||
|
|
and thus may change what DataHub users are generated by the Okta source. And in a follow up <code>okta_profile_to_username_regex</code> has been set to <code>.*</code> which taken together with previous change brings the defaults in line with OIDC.</li><li>#8331: For all sql-based sources that support profiling, you can no longer specify
|
|||
|
|
<code>profile_table_level_only</code> together with <code>include_field_xyz</code> config options to ingest
|
|||
|
|
certain column-level metrics. Instead, set <code>profile_table_level_only</code> to <code>false</code> and
|
|||
|
|
individually enable / disable desired field metrics.</li><li>#8451: The <code>bigquery-beta</code> and <code>snowflake-beta</code> source aliases have been dropped. Use <code>bigquery</code> and <code>snowflake</code> as the source type instead.</li><li>#8472: Ingestion runs created with Pipeline.create will show up in the DataHub ingestion tab as CLI-based runs. To revert to the previous behavior of not showing these runs in DataHub, pass <code>no_default_report=True</code>.</li><li>#8513: <code>snowflake</code> connector will use user's <code>email</code> attribute as is in urn. To revert to previous behavior disable <code>email_as_user_identifier</code> in recipe.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-12">Potential Downtime<a href="#potential-downtime-12" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><ul><li>BrowsePathsV2 upgrade will now be handled by the <code>system-update</code> job in non-blocking mode. This process generates data needed for the new search
|
|||
|
|
and browse feature. This process must complete before enabling the new search and browse UI and while upgrading entities will be missing from the UI.
|
|||
|
|
If not using the new search and browse UI, there will be no impact and the update will complete in the background.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-12">Deprecations<a href="#deprecations-12" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><ul><li>#8198: In the Python SDK, the <code>PlatformKey</code> class has been renamed to <code>ContainerKey</code>.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-11">Other Notable Changes<a href="#other-notable-changes-11" class="hash-link" aria-label="Direct link to Other Notable Changes" title="Direct link to Other Notable Changes"></a></h3><p>0.10.5 introduces the new Unified Search & Browse experience and is disabled by default. You can control whether or not you want to see just the new search filtering experience, the new search and browse experience together, or keep the existing search and browse experiences by toggling the two environment variable feature flags <code>SHOW_SEARCH_FILTERS_V2</code> and <code>SHOW_BROWSE_V2</code> in your GMS container.</p><p><strong>Upgrade Considerations:</strong></p><ul><li>With the release of Browse V2, we have created a job to run in GMS that will backfill your existing data with new <code>browsePathsV2</code> aspects. This job loops over entity types that need a <code>browsePathsV2</code> aspect (Dataset, Dashboard, Chart, DataJob, DataFlow, MLModel, MLModelGroup, MLFeatureTable, and MLFeature) and generates one for them. For entities that may have Container parents (Datasets and Dashboards) we will try to fetch their parent containers in order to generate this new aspect. For those deployments with large amounts of data, consider whether running this upgrade job makes sense as it may be a heavy operation and take some time to complete. If you wish to skip this job, simply set the <code>BACKFILL_BROWSE_PATHS_V2</code> environment variable flag to <code>false</code> in your GMS container. Without this backfill job, though, you will need to rely on the newest CLI of ingestion to create these <code>browsePathsV2</code> aspects when running ingestion otherwise your browse sidebar will be out-of-sync.</li><li>Since the new browse experience replaces the old, consider whether having the <code>SHOW_BROWSE_V2</code> environment variable feature flag on is the right decision for your organization. If you’re creating custom browse paths with the <code>browsePaths</code> aspect, you can continue to do the same with the new experience, however you will have to generate <code>browsePathsV2</code> aspects instead which are documented <a href="/docs/browsev2/browse-paths-v2/">here</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="0104">0.10.4<a href="#0104" class="hash-link" aria-label="Direct link to 0.10.4" title="Direct link to 0.10.4"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-13">Breaking Changes<a href="#breaking-changes-13" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-13">Potential Downtime<a href="#potential-downtime-13" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-13">Deprecations<a href="#deprecations-13" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><ul><li>#8045: With the introduction of custom ownership types, the <code>Owner</code> aspect has been updated where the <code>type</code> field is deprecated in favor of a new field <code>typeUrn</code>. This latter field is an urn reference to the new OwnershipType entity. GraphQL endpoints have been updated to use the new field. For pre-existing ownership aspect records, DataHub now has logic to map the old field to the new field.</li></ul><h3 class="anchor anchorWithStic
|
|||
|
|
the repository. Please refer to older releases if needed.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="0100">0.10.0<a href="#0100" class="hash-link" aria-label="Direct link to 0.10.0" title="Direct link to 0.10.0"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-16">Breaking Changes<a href="#breaking-changes-16" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li>#7103 This should only impact users who have configured explicit non-default names for DataHub's Kafka topics. The environment variables used to configure Kafka topics for DataHub used in the <code>kafka-setup</code> docker image have been updated to be in-line with other DataHub components, for more info see our docs on <a href="https://docs.datahub.com/docs/how/kafka-config" target="_blank" rel="noopener noreferrer">Configuring Kafka in DataHub
|
|||
|
|
</a>. They have been suffixed with <code>_TOPIC</code> where as now the correct suffix is <code>_TOPIC_NAME</code>. This change should not affect any user who is using default Kafka names.</li><li>#6906 The Redshift source has been reworked and now also includes usage capabilities. The old Redshift source was renamed to <code>redshift-legacy</code>. The <code>redshift-usage</code> source has also been renamed to <code>redshift-usage-legacy</code> will be removed in the future.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-15">Potential Downtime<a href="#potential-downtime-15" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><ul><li>#6894 Search improvements requires reindexing indices. A <code>system-update</code> job will run which will set indices to read-only and create a backup/clone of each index. During the reindexing new components will be prevented from start-up until the reindex completes. The logs of this job will indicate a % complete per index. Depending on index sizes and infrastructure this process can take 5 minutes to hours however as a rough estimate 1 hour for every 2.3 million entities.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="helm-notes">Helm Notes<a href="#helm-notes" class="hash-link" aria-label="Direct link to Helm Notes" title="Direct link to Helm Notes"></a></h4><p>Helm without <code>--atomic</code>: The default timeout for an upgrade command is 5 minutes. If the reindex takes longer (depending on data size) it will continue to run in the background even though helm will report a failure. Allow this job to finish and then re-run the helm upgrade command.</p><p>Helm with <code>--atomic</code>: In general, it is recommended to not use the <code>--atomic</code> setting for this particular upgrade since the system update job will be terminated before completion. If <code>--atomic</code> is preferred, then increase the timeout using the <code>--timeout</code> flag to account for the reindexing time (see note above for estimating this value).</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-16">Deprecations<a href="#deprecations-16" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><h2 class="anchor anchorWithStickyNavbar_LWe7" id="096">0.9.6<a href="#096" class="hash-link" aria-label="Direct link to 0.9.6" title="Direct link to 0.9.6"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-17">Breaking Changes<a href="#breaking-changes-17" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li>#6742 The metadata file sink's output format no longer contains nested JSON strings for MCP aspects, but instead unpacks the stringified JSON into a real JSON object. The previous sink behavior can be recovered using the <code>legacy_nested_json_string</code> option. The file source is backwards compatible and supports both formats.</li><li>#6901 The <code>env</code> and <code>database_alias</code> fields have been marked deprecated across all sources. We recommend using <code>platform_instance</code> where possible instead.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-16">Potential Downtime<a href="#potential-downtime-16" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-17">Deprecations<a href="#deprecations-17" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><ul><li>#6851 - Sources bigquery-legacy and bigquery-usage-legacy have been removed</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-14">Other notable Changes<a href="#other-notable-changes-14" class="hash-link" aria-label="Direct link to Other notable Changes" title="Direct link to Other notable Changes">
|
|||
|
|
<code>getInviteToken</code> and <code>createInviteToken</code> respectively. Additionally, both now accept an optional <code>roleUrn</code> parameter.
|
|||
|
|
Both endpoints also now require the <code>MANAGE_POLICIES</code> privilege to execute, rather than <code>MANAGE_USER_CREDENTIALS</code>
|
|||
|
|
privilege.</li><li>One of the default policies shipped with DataHub (<code>urn:li:dataHubPolicy:7</code>, or <code>All Users - All Platform Privileges</code>)
|
|||
|
|
has been edited to no longer include <code>MANAGE_POLICIES</code>. Its name has consequently been changed to
|
|||
|
|
<code>All Users - All Platform Privileges (EXCEPT MANAGE POLICIES)</code>. This change was made to prevent all users from
|
|||
|
|
effectively acting as superusers by default.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-22">Potential Downtime<a href="#potential-downtime-22" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-23">Deprecations<a href="#deprecations-23" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-20">Other notable Changes<a href="#other-notable-changes-20" class="hash-link" aria-label="Direct link to Other notable Changes" title="Direct link to Other notable Changes"></a></h3><h2 class="anchor anchorWithStickyNavbar_LWe7" id="v0844"><code>v0.8.44</code><a href="#v0844" class="hash-link" aria-label="Direct link to v0844" title="Direct link to v0844"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-24">Breaking Changes<a href="#breaking-changes-24" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li>Browse Paths have been upgraded to a new format to align more closely with the intention of the feature.
|
|||
|
|
Learn more about the changes, including steps on upgrading, here: <a href="https://docs.datahub.com/docs/advanced/browse-paths-upgrade" target="_blank" rel="noopener noreferrer">https://docs.datahub.com/docs/advanced/browse-paths-upgrade</a></li><li>The dbt ingestion source's <code>disable_dbt_node_creation</code> and <code>load_schema</code> options have been removed. They were no longer necessary due to the recently added sibling entities functionality.</li><li>The <code>snowflake</code> source now uses newer faster implementation (earlier <code>snowflake-beta</code>). Config properties <code>provision_role</code> and <code>check_role_grants</code> are not supported. Older <code>snowflake</code> and <code>snowflake-usage</code> are available as <code>snowflake-legacy</code> and <code>snowflake-usage-legacy</code> sources respectively.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-23">Potential Downtime<a href="#potential-downtime-23" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><ul><li>[Helm]<!-- --> If you're using Helm, please ensure that your version of the <code>datahub-actions</code> container is bumped to <code>v0.0.7</code> or <code>head</code>.
|
|||
|
|
This version contains changes to support running ingestion in debug mode. Previous versions are not compatible with this release.
|
|||
|
|
Upgrading to helm chart version <code>0.2.103</code> will ensure that you have the compatible versions by default.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-24">Deprecations<a href="#deprecations-24" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-21">Other notable Changes<a href="#other-notable-changes-21" class="hash-link" aria-label="Direct link to Other notable Changes" title="Direct link to Other notable Changes"></a></h3><h2 class="anchor anchorWithStickyNavbar_LWe7" id="v0842"><code>v0.8.42</code><a href="#v0842" class="hash-link" aria-label="Direct link to v0842" title="Direct link to v0842"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-25">Breaking Changes<a href="#breaking-changes-25" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li>Python 3.6 is no longer supported for metadata ingestion</li><li>#5451 <code>GMS_HOST</code> and <code>GMS_PORT</code> environment variables deprecated in <code>v0.8.39</code> have been removed. Use <code>DATAHUB_GMS_HOST</code> and <code>DATAHUB_GMS_PORT</code> instead.</li><li>#5478 DataHub CLI <code>delete</code> command when used with <code>--hard</code> option will delete soft-deleted entities which match the other filters given.</li><li>#5471 Looker now populates <code>userEmail</code> in dashboard user usage stats. This version of looker connnector will not work with older version of <strong>datahub-gms</strong> if you have <code>extract_usage_history</code> looker config enabled.</li><li>#5529 - <code>ANALYTICS_ENABLED</code> environment variable in <strong>datahub-gms</strong> is now deprecated. Use <code>DATAHUB_ANALYTICS_ENABLED</code> instead.</li><li>#5485 <code>--include-removed</code> option was removed from delete CLI</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-24">Potential Downtime<a href="#potential-downtime-24" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-25">Deprecations<a href="#deprecations-25" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-22">Other notable Changes<a href="#other-notable-changes-22" class="hash-link" aria-label="Direct link to Other notable Changes" title="Direct link to Other notable Changes"></a></h3><h2 class="anchor anchorWithStickyNavbar_LWe7" id="v0841"><code>v0.8.41</code><a href="#v0841" class="hash-link" aria-label="Direct link to v0841" title="Direct link to v0841"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-26">Breaking Changes<a href="#breaking-changes-26" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li><p>The <code>should_overwrite</code> flag in <code>csv-enricher</code> has been replaced with <code>write_semantics</code> to match the format used for other sources. See the <a href="/docs/generated/ingestion/sources/csv-enricher/">documentation</a> for more details</p></li><li><p>Closing an authorization hole in creating tags adding a Platform Privilege called <code>Create Tags</code> for creating tags. This is assigned to <code>datahub</code> root user, along
|
|||
|
|
with default All Users policy. Notice: You may need to add this privilege (or <code>Manage Tags</code>) to existing users that need the ability to create tags on the platform.</p></li><li><p>#5329 Below profiling config parameters are now supported in <code>BigQuery</code>:</p><ul><li>profiling.profile_if_updated_since_days (default=1)</li><li>profiling.profile_table_size_limit (default=1GB)</li><li>profiling.profile_table_row_limit (default=50000)</li></ul><p>Set above parameters to <code>null</code> if you want older behaviour.</p></li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-25">Potential Downtime<a href="#potential-downtime-25" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-26">Deprecations<a href="#deprecations-26" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-23">Other notable Changes<a href="#other-notable-changes-23" class="hash-link" aria-label="Direct link to Other notable Changes" title="Direct link to Other notable Changes"></a></h3><h2 class="anchor anchorWithStickyNavbar_LWe7" id="v0840"><code>v0.8.40</code><a href="#v0840" class="hash-link" aria-label="Direct link to v0840" title="Direct link to v0840"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-27">Breaking Changes<a href="#breaking-changes-27" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li>#5240 <code>lineage_client_project_id</code> in <code>bigquery</code> source is removed. Use <code>storage_project_id</code> instead.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-26">Potential Downtime<a href="#potential-downtime-26" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-27">Deprecations<a href="#deprecations-27" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="other-notable-changes-24">Other notable Changes<a href="#other-notable-changes-24" class="hash-link" aria-label="Direct link to Other notable Changes" title="Direct link to Other notable Changes"></a></h3><h2 class="anchor anchorWithStickyNavbar_LWe7" id="v0839"><code>v0.8.39</code><a href="#v0839" class="hash-link" aria-label="Direct link to v0839" title="Direct link to v0839"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaking-changes-28">Breaking Changes<a href="#breaking-changes-28" class="hash-link" aria-label="Direct link to Breaking Changes" title="Direct link to Breaking Changes"></a></h3><ul><li>Refactored the <code>health</code> field of the <code>Dataset</code> GraphQL Type to be of type <strong>list of HealthStatus</strong> (was type <strong>HealthStatus</strong>). See <a href="https://github.com/datahub-project/datahub/pull/5222/files" target="_blank" rel="noopener noreferrer">this PR</a> for more details.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="potential-downtime-27">Potential Downtime<a href="#potential-downtime-27" class="hash-link" aria-label="Direct link to Potential Downtime" title="Direct link to Potential Downtime"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations-28">Deprecations<a href="#deprecations-28" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3><ul><li>#4875 Lookml view file contents will no longer be populated in custom_properties, instead view definitions will be always available in the View Definitions tab.</li><li>#5208 <code>GMS_HOST</code> and <code>GMS_PORT</code> environment variables being set in various containers are deprecated in favour of <code>DA
|
|||
|
|
<script src="/assets/js/runtime~main.310f59c4.js"></script>
|
|||
|
|
<script src="/assets/js/main.49198d73.js"></script>
|
|||
|
|
</body>
|
|||
|
|
</html>
|