yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-07-23 09:32:04 +00:00
Code Issues Packages Projects Releases Wiki Activity
datahub/docs/ui-ingestion/index.html
chriscollins3456 1631a2786f deploy: 1350fe21fa7433529c469abc27de800166a57a53
2025-07-17 21:02:20 +00:00

110 lines
84 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!doctype html>
<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-docs/ui-ingestion" data-has-hydrated="false">
<head>
<meta charset="UTF-8">
<meta name="generator" content="Docusaurus v2.4.3">
<title data-rh="true">Metadata Ingestion | DataHub</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://docs.datahub.com/docs/ui-ingestion"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Metadata Ingestion | DataHub"><meta data-rh="true" name="description" content="DataHub helps you discover and understand your organization&#x27;s data by automatically collecting information about your data sources. This process is called metadata ingestion, allowing DataHub to automatically pull in:"><meta data-rh="true" property="og:description" content="DataHub helps you discover and understand your organization&#x27;s data by automatically collecting information about your data sources. This process is called metadata ingestion, allowing DataHub to automatically pull in:"><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://docs.datahub.com/docs/ui-ingestion"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/ui-ingestion" hreflang="en"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/ui-ingestion" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://RK0UG797F3-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/learn/rss.xml" title="DataHub RSS Feed">
<link rel="alternate" type="application/atom+xml" href="/learn/atom.xml" title="DataHub Atom Feed">
<link rel="preconnect" href="https://www.google-analytics.com">
<link rel="preconnect" href="https://www.googletagmanager.com">
<script async src="https://www.googletagmanager.com/gtag/js?id=G-PKGVLETT4C"></script>
<script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-PKGVLETT4C",{})</script>
<link rel="preconnect" href="https://www.googletagmanager.com">
<script>window.dataLayer=window.dataLayer||[]</script>
<script>!function(e,t,a,n,g){e[n]=e[n]||[],e[n].push({"gtm.start":(new Date).getTime(),event:"gtm.js"});var m=t.getElementsByTagName(a)[0],r=t.createElement(a);r.async=!0,r.src="https://www.googletagmanager.com/gtm.js?id=GTM-5M8T9HNN",m.parentNode.insertBefore(r,m)}(window,document,"script","dataLayer")</script>
<link rel="search" type="application/opensearchdescription+xml" title="DataHub" href="/opensearch.xml">
<meta httpequiv="Content-Security-Policy" content="frame-ancestors &#39;self&#39; https://*.acryl.io https://acryldata.io http://localhost:*">
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap">
<script src="https://tools.luckyorange.com/core/lo.js?site-id=28ea8a38" async defer="defer"></script>
<script src="/scripts/rb2b.js" async defer="defer"></script>
<script src="https://app.revenuehero.io/scheduler.min.js"></script>
<script src="https://tag.clearbitscripts.com/v1/pk_2e321cabe30432a5c44c0424781aa35f/tags.js" referrerpolicy="strict-origin-when-cross-origin"></script>
<script src="/scripts/reo.js"></script>
<script id="runllm-widget-script" type="module" src="https://widget.runllm.com" crossorigin="true" runllm-name="DataHub" runllm-assistant-id="81" runllm-position="BOTTOM_RIGHT" runllm-keyboard-shortcut="Mod+j" runllm-preset="docusaurus" runllm-theme-color="#1890FF" runllm-brand-logo="https://docs.datahub.com/img/datahub-logo-color-mark.svg" runllm-community-url="https://datahub.com/slack" runllm-community-type="slack" runllm-disable-ask-a-person="true" async></script><link rel="stylesheet" href="/assets/css/styles.d8fe2eb8.css">
<link rel="preload" href="/assets/js/runtime~main.310f59c4.js" as="script">
<link rel="preload" href="/assets/js/main.49198d73.js" as="script">
</head>
<body class="navigation-with-keyboard">
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-5M8T9HNN" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:transparent;color:#ffffff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"><div class="shimmer-banner"><p>DataHub Secures $35 Million Series B</p><a href="https://datahub.com/news/series-b-announcement/" target="_blank" class="button"><div>Read the announcement<span> →</span></div></a></div></div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a href="https://datahub.com" target="_blank" rel="noopener noreferrer" class="navbar__brand"><div class="navbar__logo"><img src="/img/datahub-logo-color-light-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/datahub-logo-color-dark-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link versionNavItem_cbn8">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/ui-ingestion">Next</a></li><li><a class="dropdown__link" href="/docs/1.1.0/ui-ingestion">1.1.0</a></li><li><hr class="dropdown-separator" style="margin: 0.4rem;"></li><li><div class="dropdown__link"><b>Archived versions</b></div></li><li>
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/features">1.0.0
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
</a>
</li><li>
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/0.15.0/features">0.15.0
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
</a>
</li><li>
<a class="dropdown__link" href="https://docs-website-8jkm4uler-acryldata.vercel.app/docs/0.14.1/features">0.14.1
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
</a>
</li><li>
<a class="dropdown__link" href="https://docs-website-eue2qafvn-acryldata.vercel.app/docs/features">0.14.0
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
</a>
</li><li>
<a class="dropdown__link" href="https://docs-website-psat3nzgi-acryldata.vercel.app/docs/features">0.13.1
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
</a>
</li><li>
<a class="dropdown__link" href="https://docs-website-lzxh86531-acryldata.vercel.app/docs/features">0.13.0
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
</a>
</li><li>
<a class="dropdown__link" href="https://docs-website-2uuxmgza2-acryldata.vercel.app/docs/features">0.12.1
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
</a>
</li><li>
<a class="dropdown__link" href="https://docs-website-irpoe2osc-acryldata.vercel.app/docs/features">0.11.0
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
</a>
</li><li>
<a class="dropdown__link" href="https://docs-website-1gv2yzn9d-acryldata.vercel.app/docs/features">0.10.5
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
</a>
</li></ul></div></div><div class="navbar__items navbar__items--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/docs">Docs</a><a class="navbar__item navbar__link" href="/integrations">Integrations</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/weekly-demo" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Weekly Demo"></div><div class="title_c7DP">Weekly Demo</div></a></div><div><a href="https://datahub.com/use-cases" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-forum.png" alt="Use Cases"></div><div class="title_c7DP">Use Cases</div></a></div><div><a href="httpps://datahub.com/adoption-stories" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Adoption Stories"></div><div class="title_c7DP">Adoption Stories</div></a></div><div><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Blog"></div><div class="title_c7DP">Blog</div></a></div><div><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Youtube"></div><div class="title_c7DP">Youtube</div></a></div></div></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/slack/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Join Slack"></div><div class="title_c7DP">Join Slack</div></a></div><div><a href="https://datahub.com/events" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Events"></div><div class="title_c7DP">Events</div></a></div><div><a href="https://datahub.com/champions/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Champions"></div><div class="title_c7DP">Champions</div></a></div><div><a href="https://datahub.com/share-your-journey/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Share Your Journey"></div><div class="title_c7DP">Share Your Journey</div></a></div></div></ul></div><a href="https://datahub.com/products/why-datahub-cloud/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
<style>
.cloud-cta {
color: var(--ifm-menu-color-active);
font-weight: 600;
background: linear-gradient(40deg, var(--ifm-menu-color-active), var(--ifm-menu-color-active));
background-size: 200% 100%;
-webkit-background-clip: text;
background-clip: text;
transition: background-image 0.3s ease;
}
.cloud-cta:hover {
color: transparent;
background: linear-gradient(40deg, var(--ifm-menu-color-active), #ff1493);
background-size: 200% 100%;
-webkit-background-clip: text;
background-clip: text;
animation: gradientShift 3s ease infinite;
}
@keyframes gradientShift {
0%, 100% { background-position: 0% 50%; }
50% { background-position: 100% 50%; }
}
</style>
<div class="cloud-cta">Get Cloud</div>
</a><a href="https://datahub.com/slack?utm_source=docs&amp;utm_medium=header&amp;utm_campaign=docs_header" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
<style>
.slack-logo:hover {
opacity: 0.8;
}
</style>
<img class="slack-logo" src="https://upload.wikimedia.org/wikipedia/commons/d/d5/Slack_icon_2019.svg" , alt="slack" , height="20px" style="margin: 10px 0 0 0;">
</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Getting Started</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/features">What Is DataHub?</a><button aria-label="Toggle the collapsible sidebar category &#x27;What Is DataHub?&#x27;" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--active" aria-expanded="true" href="/docs/category/features">Features</a><button aria-label="Toggle the collapsible sidebar category &#x27;Features&#x27;" type="button" class="clean-btn menu__caret"></button></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" tabindex="0" href="/docs/managed-datahub/observe/assertions">Assertions (Data Quality)</a><button aria-label="Toggle the collapsible sidebar category &#x27;Assertions (Data Quality)&#x27;" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/features/feature-guides/access-management">Access Management</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/automations/docs-propagation">Automations</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/businessattributes">Business Attributes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/glossary/business-glossary">Business Glossary</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/features/feature-guides/compliance-forms/overview">Compliance Forms</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/managed-datahub/observe/data-contract">Data Contract</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/dataproducts">Data Products</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/features/dataset-usage-and-query-history">Dataset Usage and Query History</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/domains">Domains</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/incidents/incidents">Incidents</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/ui-ingestion">Ingestion</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" tabindex="0" href="/docs/features/feature-guides/lineage">Lineage</a><button aria-label="Toggle the collapsible sidebar category &#x27;Lineage&#x27;" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item saasOnly"><a class="menu__link" tabindex="0" href="/docs/tests/metadata-tests">Metadata Tests</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/features/feature-guides/mcp">MCP Server</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/ownership/ownership-types">Ownership</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/authorization/access-policies-guide">Policies</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/posts">Posts</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/features/feature-guides/properties/overview">Properties</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/schema-history">Schema history</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/how/search">Search</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/sync-status">Sync Status</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/tags">Tags</a></li></ul></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>DataHub Cloud</div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/managed-datahub-overview">DataHub Cloud Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/welcome-acryl">Getting Started with DataHub Cloud</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authentication/guides/sso/initialize-oidc">Configure Single Sign-On</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/remote-executor/about">Remote Executor</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/datahub-api/entity-events-api">DataHub API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/slack/saas-slack-app">Slack</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge">Operator Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item saasOnly"><a class="menu__link" href="/docs/managed-datahub/approval-workflows">Change Proposals &amp; Approval Workflows</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/chrome-extension">Cloud Chrome Extension</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item saasOnly"><a class="menu__link" href="/docs/managed-datahub/subscription-and-notification">Subscriptions &amp; Notifications</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/release-notes/v_0_3_12">DataHub Cloud Release History</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Integrations</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-ingestion">Overview</a><button aria-label="Toggle the collapsible sidebar category &#x27;Overview&#x27;" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/cli-ingestion">Quickstart Guides</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-ingestion/source_overview">Sources</a><button aria-label="Toggle the collapsible sidebar category &#x27;Sources&#x27;" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/schedule_docs/intro">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>API & SDKs</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-modeling/metadata-model">DataHub&#x27;s Open Metadata Standard</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/what-is-datahub/datahub-concepts">Concepts</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-standards">Metadata Standards</a><button aria-label="Toggle the collapsible sidebar category &#x27;Metadata Standards&#x27;" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/api/datahub-apis">APIs and SDKs Overview</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/api/graphql/overview">API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/as-a-library">Python SDK</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/metadata-integration/java/as-a-library">Java SDK</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/cli">DataHub CLI</a><button aria-label="Toggle the collapsible sidebar category &#x27;DataHub CLI&#x27;" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/act-on-metadata">DataHub Actions</a><button aria-label="Toggle the collapsible sidebar category &#x27;DataHub Actions&#x27;" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/api/tutorials/datasets">Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Admin</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authentication">Authentication</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authorization">Authorization</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/how/delete-metadata">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Deployment</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/deployment-guides">Deployment Guides</a><button aria-label="Toggle the collapsible sidebar category &#x27;Deployment Guides&#x27;" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/deploy/confluent-cloud">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Developers</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/architecture/architecture">Architecture</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/developers">Developing on DataHub</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/advanced/mcp-mcl">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Community</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/community">Community</a><button aria-label="Toggle the collapsible sidebar category &#x27;Community&#x27;" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/releases">Release History</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item"><a class="breadcrumbs__link" itemprop="item" href="/docs/category/features"><span itemprop="name">Features</span></a><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Ingestion</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Metadata Ingestion</h1><div class="availabilityCard_P5od card"><strong>Feature Availability</strong><div><span class="platform_wqXv platformAvailable_Y8lN">Self-Hosted DataHub <span role="img" aria-label="check-circle" class="anticon anticon-check-circle"><svg viewBox="64 64 896 896" focusable="false" data-icon="check-circle" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm193.5 301.7l-210.6 292a31.8 31.8 0 01-51.7 0L318.5 484.9c-3.8-5.3 0-12.7 6.5-12.7h46.9c10.2 0 19.9 4.9 25.9 13.3l71.2 98.8 157.2-218c6-8.3 15.6-13.3 25.9-13.3H699c6.5 0 10.3 7.4 6.5 12.7z"></path></svg></span></span></div><div><span role="img" aria-label="cloud" class="anticon anticon-cloud managedIcon_AxXO"><svg viewBox="64 64 896 896" focusable="false" data-icon="cloud" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M811.4 418.7C765.6 297.9 648.9 212 512.2 212S258.8 297.8 213 418.6C127.3 441.1 64 519.1 64 612c0 110.5 89.5 200 199.9 200h496.2C870.5 812 960 722.5 960 612c0-92.7-63.1-170.7-148.6-193.3zm36.3 281a123.07 123.07 0 01-87.6 36.3H263.9c-33.1 0-64.2-12.9-87.6-36.3A123.3 123.3 0 01140 612c0-28 9.1-54.3 26.2-76.3a125.7 125.7 0 0166.1-43.7l37.9-9.9 13.9-36.6c8.6-22.8 20.6-44.1 35.7-63.4a245.6 245.6 0 0152.4-49.9c41.1-28.9 89.5-44.2 140-44.2s98.9 15.3 140 44.2c19.9 14 37.5 30.8 52.4 49.9 15.1 19.3 27.1 40.7 35.7 63.4l13.8 36.5 37.8 10c54.3 14.5 92.1 63.8 92.1 120 0 33.1-12.9 64.3-36.3 87.7z"></path></svg></span><span class="platform_wqXv platformAvailable_Y8lN">DataHub Cloud <span role="img" aria-label="check-circle" class="anticon anticon-check-circle"><svg viewBox="64 64 896 896" focusable="false" data-icon="check-circle" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm193.5 301.7l-210.6 292a31.8 31.8 0 01-51.7 0L318.5 484.9c-3.8-5.3 0-12.7 6.5-12.7h46.9c10.2 0 19.9 4.9 25.9 13.3l71.2 98.8 157.2-218c6-8.3 15.6-13.3 25.9-13.3H699c6.5 0 10.3 7.4 6.5 12.7z"></path></svg></span></span></div></div><p>DataHub helps you discover and understand your organization&#x27;s data by automatically collecting information about your data sources. This process is called <strong>metadata ingestion</strong>, allowing DataHub to automatically pull in:</p><ul><li><strong>Table and column names</strong> from your databases</li><li><strong>Asset Lineage</strong> showing how information flows between systems</li><li><strong>Usage statistics</strong> revealing which datasets are most popular</li><li><strong>Data quality information</strong> including freshness and completeness</li><li><strong>Business context</strong> like ownership and documentation</li></ul><p>This makes it simple to connect to popular platforms like Snowflake, BigQuery, dbt, and more, schedule automatic updates, and manage credentials securely.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites-and-permissions">Prerequisites and Permissions<a href="#prerequisites-and-permissions" class="hash-link" aria-label="Direct link to Prerequisites and Permissions" title="Direct link to Prerequisites and Permissions">​</a></h2><p>To manage metadata ingestion in DataHub, you need appropriate permissions.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="option-1-admin-level-access">Option 1: Admin-Level Access<a href="#option-1-admin-level-access" class="hash-link" aria-label="Direct link to Option 1: Admin-Level Access" title="Direct link to Option 1: Admin-Level Access">​</a></h3><p>Users can be granted the following privileges for full administrative access to all ingestion sources:</p><ul><li><strong><code>Manage Metadata Ingestion</code></strong> - Provides complete access to create, edit, run, and delete all ingestion sources</li><li><strong><code>Manage Secrets</code></strong> - Allows creation and management of encrypted credentials used in ingestion configurations</li></ul><p>These privileges can be granted in two ways:</p><ol><li><strong>Admin Role Assignment</strong> - Users assigned to the <strong>Admin Role</strong> receive these privileges by default</li><li><strong>Custom Policy with Platform Privileges</strong> - Create a <a href="/docs/authorization/policies">Custom Policy</a> that grants the <code>Manage Metadata Ingestion</code> and <code>Manage Secrets</code> platform privileges to specific users or groups</li></ol><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/ingestion-privileges.png" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="option-2-resource-specific-policies">Option 2: Resource-Specific Policies<a href="#option-2-resource-specific-policies" class="hash-link" aria-label="Direct link to Option 2: Resource-Specific Policies" title="Direct link to Option 2: Resource-Specific Policies">​</a></h3><p>For more granular control, administrators can create <a href="/docs/authorization/policies">Custom Policies</a> that apply specifically to <strong>Ingestion Sources</strong>, allowing different users to have different levels of access:</p><ul><li><strong>View</strong> - View ingestion source configurations and run history</li><li><strong>Edit</strong> - Modify ingestion source configurations</li><li><strong>Delete</strong> - Remove ingestion sources</li><li><strong>Execute</strong> - Run ingestion sources on-demand</li></ul><p><strong>Prerequisites:</strong></p><ul><li><strong>DataHub Core</strong>: Enable the <code>VIEW_INGESTION_SOURCE_PRIVILEGES_ENABLED</code> feature flag</li><li><strong>DataHub Cloud</strong>: Work with your customer success team to get the feature enabled</li></ul><div class="theme-admonition theme-admonition-caution alert alert--warning admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 16 16"><path fill-rule="evenodd" d="M8.893 1.5c-.183-.31-.52-.5-.887-.5s-.703.19-.886.5L.138 13.499a.98.98 0 0 0 0 1.001c.193.31.53.501.886.501h13.964c.367 0 .704-.19.877-.5a1.03 1.03 0 0 0 .01-1.002L8.893 1.5zm.133 11.497H6.987v-2.003h2.039v2.003zm0-3.004H6.987V5.987h2.039v4.006z"></path></svg></span>caution</div><div class="admonitionContent_S0QG"><p><strong>Important</strong>: Once this feature flag is enabled, any policies that apply to &quot;All&quot; resource types will now include Ingestion Sources, including the default read-only policies. This will make the Ingestion tab visible and potentially actionable depending on the applied privileges. Implement this with care if you have view-only policies that should not expose the Data Sources page.</p></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="accessing-the-ingestion-interface">Accessing the Ingestion Interface<a href="#accessing-the-ingestion-interface" class="hash-link" aria-label="Direct link to Accessing the Ingestion Interface" title="Direct link to Accessing the Ingestion Interface">​</a></h3><p>Once you have the appropriate privileges, navigate to the <strong>Ingestion</strong> tab in DataHub.</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/ingestion-tab.png" class="img_ev3q"></p><p>On this page, you&#x27;ll see a list of active <strong>Ingestion Sources</strong>. An Ingestion Source represents a configured connection to an external data system from which DataHub extracts metadata.</p><p>If you&#x27;re just getting started, you won&#x27;t have any sources configured. The following sections will guide you through creating your first ingestion source.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="creating-an-ingestion-source">Creating an Ingestion Source<a href="#creating-an-ingestion-source" class="hash-link" aria-label="Direct link to Creating an Ingestion Source" title="Direct link to Creating an Ingestion Source">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-select-a-data-source">Step 1: Select a Data Source<a href="#step-1-select-a-data-source" class="hash-link" aria-label="Direct link to Step 1: Select a Data Source" title="Direct link to Step 1: Select a Data Source">​</a></h3><p>Begin by clicking <strong>+ Create new source</strong> to start the ingestion source creation process.</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/create-new-ingestion-source-button.png" class="img_ev3q"></p><p>Next, select the type of data source you want to connect. DataHub provides pre-built templates for popular platforms including:</p><ul><li><strong>Data Warehouses</strong>: Snowflake, BigQuery, Redshift, Databricks</li><li><strong>Databases</strong>: MySQL, PostgreSQL, SQL Server, Oracle</li><li><strong>Business Intelligence</strong>: Looker, Tableau, PowerBI</li><li><strong>Streaming</strong>: Kafka, Pulsar</li><li><strong>And many more...</strong></li></ul><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/select-platform-template.png" class="img_ev3q"></p><p>Select the template that matches your data source. If your specific platform isn&#x27;t listed, you can choose <strong>Custom</strong> to configure a source manually, though this requires more technical knowledge.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-configure-connection-details">Step 2: Configure Connection Details<a href="#step-2-configure-connection-details" class="hash-link" aria-label="Direct link to Step 2: Configure Connection Details" title="Direct link to Step 2: Configure Connection Details">​</a></h3><p>After selecting your data source template, you&#x27;ll be presented with a user-friendly form to configure the connection. The exact fields will vary depending on your chosen platform, but typically include:</p><p><strong>Connection Information:</strong></p><ul><li>Host/server address and port</li><li>Database or project names</li><li>Authentication credentials</li></ul><p><strong>Data Selection:</strong></p><ul><li>Which databases, schemas, or tables to include</li><li>Filtering options to exclude certain data</li><li>Sampling and profiling settings</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="managing-sensitive-information-with-secrets">Managing Sensitive Information with Secrets<a href="#managing-sensitive-information-with-secrets" class="hash-link" aria-label="Direct link to Managing Sensitive Information with Secrets" title="Direct link to Managing Sensitive Information with Secrets">​</a></h4><p>For production environments, sensitive information like passwords and API keys should be stored securely using DataHub&#x27;s <strong>Secrets</strong> functionality.</p><p>To create a secret:</p><ol><li>Navigate to the <strong>Secrets</strong> tab in the Ingestion interface</li><li>Click <strong>Create new secret</strong></li></ol><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/create-secret.png" class="img_ev3q"></p><ol start="3"><li>Provide a descriptive name (e.g., <code>BIGQUERY_PRIVATE_KEY</code>)</li><li>Enter the sensitive value</li><li>Optionally add a description</li><li>Click <strong>Create</strong></li></ol><p>Once created, secrets can be referenced in your ingestion configuration forms using the dropdown menus provided for credential fields.</p><blockquote><p><strong>Security Note</strong>: Users with the <code>Manage Secrets</code> privilege can retrieve plaintext secret values through DataHub&#x27;s GraphQL API. Ensure secrets are only accessible to trusted administrators.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-test-your-connection">Step 3: Test Your Connection<a href="#step-3-test-your-connection" class="hash-link" aria-label="Direct link to Step 3: Test Your Connection" title="Direct link to Step 3: Test Your Connection">​</a></h3><p>Before proceeding, it&#x27;s important to verify that DataHub can successfully connect to your data source. Most ingestion source forms include a <strong>Test Connection</strong> button that validates:</p><ul><li>Network connectivity to your data source</li><li>Authentication credentials</li><li>Required permissions for metadata extraction</li></ul><p align="center"><img loading="lazy" width="75%" alt="Test BigQuery connection" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/guides/bigquery/bigquery-test-connection.png" class="img_ev3q"></p><p>If the connection test fails, review your configuration and ensure that:</p><ul><li>Network access is available between DataHub and your data source</li><li>Credentials are correct and have sufficient permissions</li><li>Any firewall rules allow the connection</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-schedule-execution-optional">Step 4: Schedule Execution (Optional)<a href="#step-4-schedule-execution-optional" class="hash-link" aria-label="Direct link to Step 4: Schedule Execution (Optional)" title="Direct link to Step 4: Schedule Execution (Optional)">​</a></h3><p>You can configure automatic execution of your ingestion source on a regular schedule. This ensures your metadata stays up-to-date without manual intervention.</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/schedule-ingestion.png" class="img_ev3q"></p><p>If you prefer to run ingestion manually or on an ad-hoc basis, you can skip the scheduling step entirely.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-finish-up-and-run">Step 5: Finish Up and Run<a href="#step-5-finish-up-and-run" class="hash-link" aria-label="Direct link to Step 5: Finish Up and Run" title="Direct link to Step 5: Finish Up and Run">​</a></h3><p>Finally, provide a descriptive name for your ingestion source that will help you and your team identify it later.</p><p>You can also assign <strong>Users</strong> and/or <strong>Groups</strong> as owners of this ingestion source. By default, you (the creator) will be assigned as an owner, but you can add additional owners or change this at any time after creation.</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/name-ingestion-source.png" class="img_ev3q"></p><p>Click <strong>Save and Run</strong> to create the ingestion source and execute it immediately, or <strong>Save</strong> to create it without running.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="advanced-configuration-options">Advanced Configuration Options<a href="#advanced-configuration-options" class="hash-link" aria-label="Direct link to Advanced Configuration Options" title="Direct link to Advanced Configuration Options">​</a></h4><p>For users who need additional control, DataHub provides advanced configuration options:</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/custom-ingestion-cli-version.png" class="img_ev3q"></p><ul><li><strong>CLI Version:</strong> Specify a particular version of the DataHub CLI for ingestion execution</li><li><strong>Environment Variables:</strong> Set custom environment variables for the ingestion process</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="running-and-monitoring-ingestion">Running and Monitoring Ingestion<a href="#running-and-monitoring-ingestion" class="hash-link" aria-label="Direct link to Running and Monitoring Ingestion" title="Direct link to Running and Monitoring Ingestion">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="executing-an-ingestion-source">Executing an Ingestion Source<a href="#executing-an-ingestion-source" class="hash-link" aria-label="Direct link to Executing an Ingestion Source" title="Direct link to Executing an Ingestion Source">​</a></h3><p>Once you&#x27;ve created your Ingestion Source, you can run it by clicking the &#x27;Play&#x27; button. Shortly after, you should see the &#x27;Last Status&#x27; column of the ingestion source change to <code>Running</code>, indicating that DataHub has successfully queued the ingestion job.</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/ingestion/running.png" class="img_ev3q"></p><p>When ingestion completes successfully, the status will show as <code>Success</code> in green.</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/ingestion/success-run.png" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="viewing-run-history">Viewing Run History<a href="#viewing-run-history" class="hash-link" aria-label="Direct link to Viewing Run History" title="Direct link to Viewing Run History">​</a></h3><p>The <strong>Run History</strong> tab shows you a complete history of all your ingestion runs. Here you can:</p><ul><li><strong>See all runs</strong>: View every ingestion execution across all your sources</li><li><strong>Check recent activity</strong>: Runs are listed with the most recent at the top</li><li><strong>Filter by source</strong>: Use the dropdown to see runs from a specific ingestion source</li><li><strong>Access from Sources tab</strong>: Click on any source&#x27;s <strong>Last Run</strong> status or select <strong>View Run History</strong> from the source menu</li></ul><p>This makes it easy to track your ingestion performance and troubleshoot any issues over time.</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/ingestion/run-history-tab.png" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="viewing-ingestion-results">Viewing Ingestion Results<a href="#viewing-ingestion-results" class="hash-link" aria-label="Direct link to Viewing Ingestion Results" title="Direct link to Viewing Ingestion Results">​</a></h3><p>After successful ingestion, you can view detailed information about what was extracted:</p><ol><li>Click the <strong>Success</strong> status button on a completed ingestion run</li><li>Select <strong>View All</strong> to see the list of ingested entities</li><li>Click on individual entities to validate the extracted metadata</li></ol><p align="center"><img loading="lazy" width="75%" alt="ingestion_details_view_all" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/guides/bigquery/bigquery_ingestion_details_view_all.png" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cancelling-running-ingestion">Cancelling Running Ingestion<a href="#cancelling-running-ingestion" class="hash-link" aria-label="Direct link to Cancelling Running Ingestion" title="Direct link to Cancelling Running Ingestion">​</a></h3><p>If an ingestion run is taking too long or appears to be stuck, you can cancel it by clicking the &#x27;Stop&#x27; button on the running job.</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/ingestion/cancelled-run.png" class="img_ev3q"></p><p>This is useful when encountering issues like:</p><ul><li>Network timeouts</li><li>Ingestion source bugs</li><li>Resource constraints</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="troubleshooting-failed-ingestion">Troubleshooting Failed Ingestion<a href="#troubleshooting-failed-ingestion" class="hash-link" aria-label="Direct link to Troubleshooting Failed Ingestion" title="Direct link to Troubleshooting Failed Ingestion">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-failure-reasons">Common Failure Reasons<a href="#common-failure-reasons" class="hash-link" aria-label="Direct link to Common Failure Reasons" title="Direct link to Common Failure Reasons">​</a></h3><p>When ingestion fails, the most common causes include:</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/ingestion/failed-source.png" class="img_ev3q"></p><ol><li><strong>Configuration Errors</strong>: Incorrect connection details, missing required fields, or invalid parameter values</li><li><strong>Authentication Issues</strong>: Wrong credentials, expired tokens, or insufficient permissions</li><li><strong>Network Connectivity</strong>: DNS resolution failures, firewall blocks, or unreachable data sources</li><li><strong>Secret Resolution Problems</strong>: Referenced secrets that don&#x27;t exist or have incorrect names</li><li><strong>Resource Constraints</strong>: Memory limits, timeouts, or processing capacity issues</li></ol><h3 class="anchor anchorWithStickyNavbar_LWe7" id="viewing-detailed-logs">Viewing Detailed Logs<a href="#viewing-detailed-logs" class="hash-link" aria-label="Direct link to Viewing Detailed Logs" title="Direct link to Viewing Detailed Logs">​</a></h3><p>To diagnose ingestion failures, click on a run history status (Failed, Aborted) value to view and download comprehensive ingestion run logs.</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/ingestion-logs.png" class="img_ev3q"></p><p>The logs provide detailed information about:</p><ul><li>Connection attempts and errors</li><li>Authentication failures</li><li>Data extraction progress</li><li>Error messages and stack traces</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="authentication-for-secured-datahub-instances">Authentication for Secured DataHub Instances<a href="#authentication-for-secured-datahub-instances" class="hash-link" aria-label="Direct link to Authentication for Secured DataHub Instances" title="Direct link to Authentication for Secured DataHub Instances">​</a></h3><p>If your DataHub instance has <a href="/docs/authentication/introducing-metadata-service-authentication">Metadata Service Authentication</a> enabled, you&#x27;ll need to provide a Personal Access Token in your configuration.</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/ingestion-with-token.png" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="advanced-configuration-with-yaml">Advanced Configuration with YAML<a href="#advanced-configuration-with-yaml" class="hash-link" aria-label="Direct link to Advanced Configuration with YAML" title="Direct link to Advanced Configuration with YAML">​</a></h2><p>While the UI-based forms handle most common ingestion scenarios, advanced users may need direct access to YAML configuration for:</p><ul><li>Custom ingestion sources not available in the UI</li><li>Complex transformation pipelines</li><li>Advanced filtering and processing logic</li><li>Integration with external systems</li></ul><p>For these advanced use cases, DataHub supports direct YAML recipe configuration. For detailed information about YAML-based configuration, including syntax and examples, see the <a href="/docs/metadata-ingestion/recipe_overview">Recipe Overview Guide</a>.</p><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">CLI</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">GraphQL</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><p>You can deploy recipes using the CLI as mentioned in the <a href="/docs/cli#ingest-deploy">CLI documentation for uploading ingestion recipes</a>.</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub ingest deploy --name </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;My Test Ingestion Source&quot;</span><span class="token plain"> --schedule </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;5 * * * *&quot;</span><span class="token plain"> --time-zone </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;UTC&quot;</span><span class="token plain"> -c recipe.yaml</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><p>Create ingestion sources using <a href="/docs/api/graphql/overview">DataHub&#x27;s GraphQL API</a> using the <strong>createIngestionSource</strong> mutation endpoint.</p><div class="language-graphql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-graphql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">mutation</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property-query property-mutation property-mutation">createIngestionSource</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">input</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token description string" style="color:rgb(195, 232, 141)">&quot;</span><span class="token description string language-markdown" style="color:rgb(195, 232, 141)">My Test Ingestion Source</span><span class="token description string" style="color:rgb(195, 232, 141)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">type</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token description string" style="color:rgb(195, 232, 141)">&quot;</span><span class="token description string language-markdown" style="color:rgb(195, 232, 141)">mysql</span><span class="token description string" style="color:rgb(195, 232, 141)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">description</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token description string" style="color:rgb(195, 232, 141)">&quot;</span><span class="token description string language-markdown" style="color:rgb(195, 232, 141)">My ingestion source description</span><span class="token description string" style="color:rgb(195, 232, 141)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">schedule</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">interval</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;*/5 * * * *&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">timezone</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;UTC&quot;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">config</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">recipe</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token description string" style="color:rgb(195, 232, 141)">&quot;</span><span class="token description string language-markdown" style="color:rgb(195, 232, 141)">{\&quot;source\&quot;:{\&quot;type\&quot;:\&quot;mysql\&quot;,\&quot;config\&quot;:{\&quot;include_tables\&quot;:true,\&quot;database\&quot;:null,\&quot;password\&quot;:\&quot;${MYSQL_PASSWORD}\&quot;,\&quot;profiling\&quot;:{\&quot;enabled\&quot;:false},\&quot;host_port\&quot;:null,\&quot;include_views\&quot;:true,\&quot;username\&quot;:\&quot;${MYSQL_USERNAME}\&quot;}},\&quot;pipeline_name\&quot;:\&quot;urn:li:dataHubIngestionSource:f38bd060-4ea8-459c-8f24-a773286a2927\&quot;}</span><span class="token description string" style="color:rgb(195, 232, 141)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">version</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token description string" style="color:rgb(195, 232, 141)">&quot;</span><span class="token description string language-markdown" style="color:rgb(195, 232, 141)">0.8.18</span><span class="token description string" style="color:rgb(195, 232, 141)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">executorId</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;mytestexecutor&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><strong>Note</strong>: Recipe must be double quotes escaped when using GraphQL</p></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="frequently-asked-questions">Frequently Asked Questions<a href="#frequently-asked-questions" class="hash-link" aria-label="Direct link to Frequently Asked Questions" title="Direct link to Frequently Asked Questions">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="why-does-ingestion-fail-with-failed-to-connect-errors-in-docker-environments">Why does ingestion fail with &#x27;Failed to Connect&#x27; errors in Docker environments?<a href="#why-does-ingestion-fail-with-failed-to-connect-errors-in-docker-environments" class="hash-link" aria-label="Direct link to Why does ingestion fail with &#x27;Failed to Connect&#x27; errors in Docker environments?" title="Direct link to Why does ingestion fail with &#x27;Failed to Connect&#x27; errors in Docker environments?">​</a></h3><p>If you&#x27;re running DataHub using <code>datahub docker quickstart</code> and experiencing connection failures, this may be due to network configuration issues. The ingestion executor might be unable to reach DataHub&#x27;s backend services.</p><p>Try updating your ingestion configuration to use the Docker internal DNS name:</p><p align="center"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/quickstart-ingestion-config.png" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="what-does-a-dash-mark---status-mean-and-how-do-i-fix-it">What does a dash mark (-) status mean and how do I fix it?<a href="#what-does-a-dash-mark---status-mean-and-how-do-i-fix-it" class="hash-link" aria-label="Direct link to What does a dash mark (-) status mean and how do I fix it?" title="Direct link to What does a dash mark (-) status mean and how do I fix it?">​</a></h3><p>If your ingestion source shows a dash mark (-) status and never changes to &#x27;Running&#x27;, this could mean:</p><ol><li><strong>The source has never been triggered to run</strong> - Try clicking the &quot;Play&quot; button to execute the source</li><li><strong>The DataHub actions executor is not running or healthy</strong> (DataHub Core users only)</li></ol><p>If clicking &quot;Play&quot; doesn&#x27;t resolve the issue, DataHub Core users should diagnose their actions container:</p><ol><li>Check container status with <code>docker ps</code></li><li>View executor logs with <code>docker logs &lt;container-id&gt;</code></li><li>Restart the actions container if necessary</li></ol><h3 class="anchor anchorWithStickyNavbar_LWe7" id="when-should-i-use-cliyaml-instead-of-ui-ingestion">When should I use CLI/YAML instead of UI ingestion?<a href="#when-should-i-use-cliyaml-instead-of-ui-ingestion" class="hash-link" aria-label="Direct link to When should I use CLI/YAML instead of UI ingestion?" title="Direct link to When should I use CLI/YAML instead of UI ingestion?">​</a></h3><p>Consider using CLI-based ingestion when:</p><ul><li>Your data sources aren&#x27;t reachable from DataHub&#x27;s network (use <a href="/docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor">remote executors</a> for DataHub Cloud)</li><li>You need custom ingestion logic not available in UI templates</li><li>Your ingestion requires local file system access</li><li>You want to distribute ingestion across multiple environments</li><li>You need complex transformations or custom metadata processing</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="additional-resources">Additional Resources<a href="#additional-resources" class="hash-link" aria-label="Direct link to Additional Resources" title="Direct link to Additional Resources">​</a></h2><ul><li><strong>Demo Video</strong>: <a href="https://www.youtube.com/watch?v=EyMyLcaw_74" target="_blank" rel="noopener noreferrer">Watch a complete UI ingestion walkthrough</a></li><li><strong>Quick Start Guides</strong>: Step-by-step setup instructions for popular data sources</li><li><strong>Recipe Documentation</strong>: <a href="/docs/metadata-ingestion/recipe_overview">Comprehensive YAML configuration reference</a></li><li><strong>Integration Catalog</strong>: <a href="/integrations">Browse all supported data sources and their features</a></li></ul></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="slackUtm_uoBr"><div class="slackUtm_uoBr"><hr>Need more help? Join the conversation in <a href="https://datahub.com/slack?utm_source=docs&amp;utm_medium=footer&amp;utm_campaign=docs_footer&amp;utm_content=docs/ui-ingestion">Slack!</a></div></div><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/datahub-project/datahub/blob/master/docs/ui-ingestion.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_VsjB"></div></div></footer><div class="feedbackWrapper_mUHF"><div class="feedbackWidget_PX4d"><div class="feedbackButtons_wn3V"><strong>Is this page helpful?</strong><div><button class="feedbackButton_UgQs"><span role="img" aria-label="like" class="anticon anticon-like"><svg viewBox="64 64 896 896" focusable="false" data-icon="like" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M885.9 533.7c16.8-22.2 26.1-49.4 26.1-77.7 0-44.9-25.1-87.4-65.5-111.1a67.67 67.67 0 00-34.3-9.3H572.4l6-122.9c1.4-29.7-9.1-57.9-29.5-79.4A106.62 106.62 0 00471 99.9c-52 0-98 35-111.8 85.1l-85.9 311H144c-17.7 0-32 14.3-32 32v364c0 17.7 14.3 32 32 32h601.3c9.2 0 18.2-1.8 26.5-5.4 47.6-20.3 78.3-66.8 78.3-118.4 0-12.6-1.8-25-5.4-37 16.8-22.2 26.1-49.4 26.1-77.7 0-12.6-1.8-25-5.4-37 16.8-22.2 26.1-49.4 26.1-77.7-.2-12.6-2-25.1-5.6-37.1zM184 852V568h81v284h-81zm636.4-353l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 16.5-7.2 32.2-19.6 43l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 16.5-7.2 32.2-19.6 43l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 22.4-13.2 42.6-33.6 51.8H329V564.8l99.5-360.5a44.1 44.1 0 0142.2-32.3c7.6 0 15.1 2.2 21.1 6.7 9.9 7.4 15.2 18.6 14.6 30.5l-9.6 198.4h314.4C829 418.5 840 436.9 840 456c0 16.5-7.2 32.1-19.6 43z"></path></svg></span></button><button class="feedbackButton_UgQs"><span role="img" aria-label="dislike" class="anticon anticon-dislike"><svg viewBox="64 64 896 896" focusable="false" data-icon="dislike" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M885.9 490.3c3.6-12 5.4-24.4 5.4-37 0-28.3-9.3-55.5-26.1-77.7 3.6-12 5.4-24.4 5.4-37 0-28.3-9.3-55.5-26.1-77.7 3.6-12 5.4-24.4 5.4-37 0-51.6-30.7-98.1-78.3-118.4a66.1 66.1 0 00-26.5-5.4H144c-17.7 0-32 14.3-32 32v364c0 17.7 14.3 32 32 32h129.3l85.8 310.8C372.9 889 418.9 924 470.9 924c29.7 0 57.4-11.8 77.9-33.4 20.5-21.5 31-49.7 29.5-79.4l-6-122.9h239.9c12.1 0 23.9-3.2 34.3-9.3 40.4-23.5 65.5-66.1 65.5-111 0-28.3-9.3-55.5-26.1-77.7zM184 456V172h81v284h-81zm627.2 160.4H496.8l9.6 198.4c.6 11.9-4.7 23.1-14.6 30.5-6.1 4.5-13.6 6.8-21.1 6.7a44.28 44.28 0 01-42.2-32.3L329 459.2V172h415.4a56.85 56.85 0 0133.6 51.8c0 9.7-2.3 18.9-6.9 27.3l-13.9 25.4 21.9 19a56.76 56.76 0 0119.6 43c0 9.7-2.3 18.9-6.9 27.3l-13.9 25.4 21.9 19a56.76 56.76 0 0119.6 43c0 9.7-2.3 18.9-6.9 27.3l-14 25.5 21.9 19a56.76 56.76 0 0119.6 43c0 19.1-11 37.5-28.8 48.4z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/incidents/incidents"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Incidents</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/features/feature-guides/lineage"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Lineage</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites-and-permissions" class="table-of-contents__link toc-highlight">Prerequisites and Permissions</a><ul><li><a href="#option-1-admin-level-access" class="table-of-contents__link toc-highlight">Option 1: Admin-Level Access</a></li><li><a href="#option-2-resource-specific-policies" class="table-of-contents__link toc-highlight">Option 2: Resource-Specific Policies</a></li><li><a href="#accessing-the-ingestion-interface" class="table-of-contents__link toc-highlight">Accessing the Ingestion Interface</a></li></ul></li><li><a href="#creating-an-ingestion-source" class="table-of-contents__link toc-highlight">Creating an Ingestion Source</a><ul><li><a href="#step-1-select-a-data-source" class="table-of-contents__link toc-highlight">Step 1: Select a Data Source</a></li><li><a href="#step-2-configure-connection-details" class="table-of-contents__link toc-highlight">Step 2: Configure Connection Details</a></li><li><a href="#step-3-test-your-connection" class="table-of-contents__link toc-highlight">Step 3: Test Your Connection</a></li><li><a href="#step-4-schedule-execution-optional" class="table-of-contents__link toc-highlight">Step 4: Schedule Execution (Optional)</a></li><li><a href="#step-5-finish-up-and-run" class="table-of-contents__link toc-highlight">Step 5: Finish Up and Run</a></li></ul></li><li><a href="#running-and-monitoring-ingestion" class="table-of-contents__link toc-highlight">Running and Monitoring Ingestion</a><ul><li><a href="#executing-an-ingestion-source" class="table-of-contents__link toc-highlight">Executing an Ingestion Source</a></li><li><a href="#viewing-run-history" class="table-of-contents__link toc-highlight">Viewing Run History</a></li><li><a href="#viewing-ingestion-results" class="table-of-contents__link toc-highlight">Viewing Ingestion Results</a></li><li><a href="#cancelling-running-ingestion" class="table-of-contents__link toc-highlight">Cancelling Running Ingestion</a></li></ul></li><li><a href="#troubleshooting-failed-ingestion" class="table-of-contents__link toc-highlight">Troubleshooting Failed Ingestion</a><ul><li><a href="#common-failure-reasons" class="table-of-contents__link toc-highlight">Common Failure Reasons</a></li><li><a href="#viewing-detailed-logs" class="table-of-contents__link toc-highlight">Viewing Detailed Logs</a></li><li><a href="#authentication-for-secured-datahub-instances" class="table-of-contents__link toc-highlight">Authentication for Secured DataHub Instances</a></li></ul></li><li><a href="#advanced-configuration-with-yaml" class="table-of-contents__link toc-highlight">Advanced Configuration with YAML</a></li><li><a href="#frequently-asked-questions" class="table-of-contents__link toc-highlight">Frequently Asked Questions</a><ul><li><a href="#why-does-ingestion-fail-with-failed-to-connect-errors-in-docker-environments" class="table-of-contents__link toc-highlight">Why does ingestion fail with &#39;Failed to Connect&#39; errors in Docker environments?</a></li><li><a href="#what-does-a-dash-mark---status-mean-and-how-do-i-fix-it" class="table-of-contents__link toc-highlight">What does a dash mark (-) status mean and how do I fix it?</a></li><li><a href="#when-should-i-use-cliyaml-instead-of-ui-ingestion" class="table-of-contents__link toc-highlight">When should I use CLI/YAML instead of UI ingestion?</a></li></ul></li><li><a href="#additional-resources" class="table-of-contents__link toc-highlight">Additional Resources</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/docs/">Introduction</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/quickstart">Quickstart</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://datahub.com/slack" target="_blank" rel="noopener noreferrer" class="footer__link-item">Slack<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="footer__link-item">YouTube<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="footer__link-item">Blog<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/docs/townhalls">Town Halls</a></li><li class="footer__item"><a href="https://datahub.com/adoption-stories/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Adoption<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://demo.datahub.com/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Demo</a></li><li class="footer__item"><a href="https://feature-requests.datahubproject.io/roadmap" target="_blank" rel="noopener noreferrer" class="footer__link-item">Roadmap<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/docs/contributing">Contributing</a></li><li class="footer__item"><a href="https://github.com/datahub-project/datahub" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://feature-requests.datahubproject.io/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Feature Requests<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2015-2025 DataHub Project Authors.</div></div></div></footer></div>
<script src="/assets/js/runtime~main.310f59c4.js"></script>
<script src="/assets/js/main.49198d73.js"></script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink