mirror of
https://github.com/datahub-project/datahub.git
synced 2025-08-07 00:37:56 +00:00
133 lines
74 KiB
HTML
133 lines
74 KiB
HTML
<!doctype html>
|
||
<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-1.1.0 plugin-docs plugin-id-default docs-doc-id-docs/how/restore-indices" data-has-hydrated="false">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="generator" content="Docusaurus v2.4.3">
|
||
<title data-rh="true">Restoring Search and Graph Indices from Local Database | DataHub</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://docs.datahub.com/docs/1.1.0/how/restore-indices"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="1.1.0"><meta data-rh="true" name="docusaurus_tag" content="docs-default-1.1.0"><meta data-rh="true" name="docsearch:version" content="1.1.0"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-1.1.0"><meta data-rh="true" property="og:title" content="Restoring Search and Graph Indices from Local Database | DataHub"><meta data-rh="true" name="description" content="If search infrastructure (Elasticsearch/Opensearch) or graph services (Elasticsearch/Opensearch/Neo4j) become inconsistent,"><meta data-rh="true" property="og:description" content="If search infrastructure (Elasticsearch/Opensearch) or graph services (Elasticsearch/Opensearch/Neo4j) become inconsistent,"><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://docs.datahub.com/docs/1.1.0/how/restore-indices"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/1.1.0/how/restore-indices" hreflang="en"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/1.1.0/how/restore-indices" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://RK0UG797F3-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/learn/rss.xml" title="DataHub RSS Feed">
|
||
<link rel="alternate" type="application/atom+xml" href="/learn/atom.xml" title="DataHub Atom Feed">
|
||
|
||
<link rel="preconnect" href="https://www.google-analytics.com">
|
||
<link rel="preconnect" href="https://www.googletagmanager.com">
|
||
<script async src="https://www.googletagmanager.com/gtag/js?id=G-PKGVLETT4C"></script>
|
||
<script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-PKGVLETT4C",{})</script>
|
||
<link rel="preconnect" href="https://www.googletagmanager.com">
|
||
<script>window.dataLayer=window.dataLayer||[]</script>
|
||
<script>!function(e,t,a,n,g){e[n]=e[n]||[],e[n].push({"gtm.start":(new Date).getTime(),event:"gtm.js"});var m=t.getElementsByTagName(a)[0],r=t.createElement(a);r.async=!0,r.src="https://www.googletagmanager.com/gtm.js?id=GTM-5M8T9HNN",m.parentNode.insertBefore(r,m)}(window,document,"script","dataLayer")</script>
|
||
|
||
|
||
<link rel="search" type="application/opensearchdescription+xml" title="DataHub" href="/opensearch.xml">
|
||
|
||
|
||
|
||
|
||
<meta httpequiv="Content-Security-Policy" content="frame-ancestors 'self' https://*.acryl.io https://acryldata.io http://localhost:*">
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap">
|
||
<script src="https://tools.luckyorange.com/core/lo.js?site-id=28ea8a38" async defer="defer"></script>
|
||
<script src="/scripts/rb2b.js" async defer="defer"></script>
|
||
<script src="https://app.revenuehero.io/scheduler.min.js"></script>
|
||
<script src="https://tag.clearbitscripts.com/v1/pk_2e321cabe30432a5c44c0424781aa35f/tags.js" referrerpolicy="strict-origin-when-cross-origin"></script>
|
||
<script src="/scripts/reo.js"></script>
|
||
<script id="runllm-widget-script" type="module" src="https://widget.runllm.com" crossorigin="true" runllm-name="DataHub" runllm-assistant-id="81" runllm-position="BOTTOM_RIGHT" runllm-keyboard-shortcut="Mod+j" runllm-preset="docusaurus" runllm-theme-color="#1890FF" runllm-brand-logo="https://docs.datahub.com/img/datahub-logo-color-mark.svg" runllm-community-url="https://datahub.com/slack" runllm-community-type="slack" runllm-disable-ask-a-person="true" async></script><link rel="stylesheet" href="/assets/css/styles.d8fe2eb8.css">
|
||
<link rel="preload" href="/assets/js/runtime~main.d9d0eca5.js" as="script">
|
||
<link rel="preload" href="/assets/js/main.75f9b227.js" as="script">
|
||
</head>
|
||
<body class="navigation-with-keyboard">
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-5M8T9HNN" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
|
||
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
|
||
<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:transparent;color:#ffffff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"><div class="shimmer-banner"><p><strong>Town Hall August 21:</strong> The Latest in DataHub Lineage</p><a href="https://events.datahub.com/august-town-hall-2025?utm_source=webinar&utm_medium=&utm_campaign=17099074-FY25-Q3-Town-Hall" target="_blank" class="button"><div>Save your spot<span> →</span></div></a></div></div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a href="https://datahub.com" target="_blank" rel="noopener noreferrer" class="navbar__brand"><div class="navbar__logo"><img src="/img/datahub-logo-color-light-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/datahub-logo-color-dark-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link versionNavItem_cbn8">1.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/how/restore-indices">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/1.1.0/how/restore-indices">1.1.0</a></li><li><hr class="dropdown-separator" style="margin: 0.4rem;"></li><li><div class="dropdown__link"><b>Archived versions</b></div></li><li>
|
||
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/features">1.0.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/0.15.0/features">0.15.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-8jkm4uler-acryldata.vercel.app/docs/0.14.1/features">0.14.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-eue2qafvn-acryldata.vercel.app/docs/features">0.14.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-psat3nzgi-acryldata.vercel.app/docs/features">0.13.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-lzxh86531-acryldata.vercel.app/docs/features">0.13.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-2uuxmgza2-acryldata.vercel.app/docs/features">0.12.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-irpoe2osc-acryldata.vercel.app/docs/features">0.11.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-1gv2yzn9d-acryldata.vercel.app/docs/features">0.10.5
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li></ul></div></div><div class="navbar__items navbar__items--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/docs">Docs</a><a class="navbar__item navbar__link" href="/integrations">Integrations</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/weekly-demo" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Weekly Demo"></div><div class="title_c7DP">Weekly Demo</div></a></div><div><a href="https://datahub.com/use-cases" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-forum.png" alt="Use Cases"></div><div class="title_c7DP">Use Cases</div></a></div><div><a href="httpps://datahub.com/adoption-stories" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Adoption Stories"></div><div class="title_c7DP">Adoption Stories</div></a></div><div><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Blog"></div><div class="title_c7DP">Blog</div></a></div><div><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Youtube"></div><div class="title_c7DP">Youtube</div></a></div></div></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/slack/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Join Slack"></div><div class="title_c7DP">Join Slack</div></a></div><div><a href="https://datahub.com/events" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Events"></div><div class="title_c7DP">Events</div></a></div><div><a href="https://datahub.com/champions/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Champions"></div><div class="title_c7DP">Champions</div></a></div><div><a href="https://datahub.com/share-your-journey/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Share Your Journey"></div><div class="title_c7DP">Share Your Journey</div></a></div></div></ul></div><a href="https://datahub.com/products/why-datahub-cloud/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
||
<style>
|
||
.cloud-cta {
|
||
color: var(--ifm-menu-color-active);
|
||
font-weight: 600;
|
||
background: linear-gradient(40deg, var(--ifm-menu-color-active), var(--ifm-menu-color-active));
|
||
background-size: 200% 100%;
|
||
-webkit-background-clip: text;
|
||
background-clip: text;
|
||
transition: background-image 0.3s ease;
|
||
}
|
||
.cloud-cta:hover {
|
||
color: transparent;
|
||
background: linear-gradient(40deg, var(--ifm-menu-color-active), #ff1493);
|
||
background-size: 200% 100%;
|
||
-webkit-background-clip: text;
|
||
background-clip: text;
|
||
animation: gradientShift 3s ease infinite;
|
||
}
|
||
@keyframes gradientShift {
|
||
0%, 100% { background-position: 0% 50%; }
|
||
50% { background-position: 100% 50%; }
|
||
}
|
||
</style>
|
||
<div class="cloud-cta">Get Cloud</div>
|
||
</a><a href="https://datahub.com/slack?utm_source=docs&utm_medium=header&utm_campaign=docs_header" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
||
<style>
|
||
.slack-logo:hover {
|
||
opacity: 0.8;
|
||
}
|
||
</style>
|
||
<img class="slack-logo" src="https://upload.wikimedia.org/wikipedia/commons/d/d5/Slack_icon_2019.svg" , alt="slack" , height="20px" style="margin: 10px 0 0 0;">
|
||
</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Getting Started</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/1.1.0/features">What Is DataHub?</a><button aria-label="Toggle the collapsible sidebar category 'What Is DataHub?'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/1.1.0/category/features">Features</a><button aria-label="Toggle the collapsible sidebar category 'Features'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/1.1.0/managed-datahub/managed-datahub-overview">DataHub Cloud</a><button aria-label="Toggle the collapsible sidebar category 'DataHub Cloud'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Integrations</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/1.1.0/metadata-ingestion">Overview</a><button aria-label="Toggle the collapsible sidebar category 'Overview'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/metadata-ingestion/cli-ingestion">Quickstart Guides</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/1.1.0/metadata-ingestion/source_overview">Sources</a><button aria-label="Toggle the collapsible sidebar category 'Sources'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/metadata-ingestion/schedule_docs/intro">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>API & SDKs</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/metadata-modeling/metadata-model">DataHub's Open Metadata Standard</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/1.1.0/what-is-datahub/datahub-concepts">Concepts</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/1.1.0/metadata-standards">Metadata Standards</a><button aria-label="Toggle the collapsible sidebar category 'Metadata Standards'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/1.1.0/api/datahub-apis">APIs</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/api/graphql/overview">API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/metadata-ingestion/as-a-library">Python SDK</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/1.1.0/metadata-integration/java/as-a-library">Java SDK</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/1.1.0/cli">DataHub CLI</a><button aria-label="Toggle the collapsible sidebar category 'DataHub CLI'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/1.1.0/act-on-metadata">DataHub Actions</a><button aria-label="Toggle the collapsible sidebar category 'DataHub Actions'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/api/tutorials/datasets">API & SDK Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Admin</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/authentication">Authentication</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/authorization">Authorization</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/1.1.0/how/delete-metadata">Advanced Guides</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.0/how/delete-metadata">Removing Metadata from DataHub</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.0/how/configuring-authorization-with-apache-ranger">Configuring Authorization with Apache Ranger</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.1.0/managed-datahub/configuring-identity-provisioning-with-ms-entra">SCIM Provisioning</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.0/how/backup-datahub">Taking backup of DataHub</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/1.1.0/how/restore-indices">Restoring Search and Graph Indices from Local Database</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.0/advanced/db-retention">Configuring Database Retention</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.0/advanced/monitoring">Monitoring DataHub</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.0/deploy/telemetry">Telemetry</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.0/how/kafka-config">Configuring Kafka</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.0/how/jattach-guide">Debugging by Jattach</a></li></ul></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Deployment</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/1.1.0/category/deployment-guides">Deployment Guides</a><button aria-label="Toggle the collapsible sidebar category 'Deployment Guides'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/deploy/confluent-cloud">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Developers</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/architecture/architecture">Architecture</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/developers">Developing on DataHub</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/advanced/mcp-mcl">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Community</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/1.1.0/category/community">Community</a><button aria-label="Toggle the collapsible sidebar category 'Community'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.0/releases">Release History</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->DataHub<!-- --> <b>1.1.0</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/docs/how/restore-indices">latest version</a></b> (<!-- -->Next<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Advanced Guides</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Restoring Search and Graph Indices from Local Database</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.1.0</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Restoring Search and Graph Indices from Local Database</h1><p>If search infrastructure (Elasticsearch/Opensearch) or graph services (Elasticsearch/Opensearch/Neo4j) become inconsistent,
|
||
you can restore them from the aspects stored in the local database.</p><p>When a new version of the aspect gets ingested, GMS initiates an MCL event for the aspect which is consumed to update
|
||
the search and graph indices. As such, we can fetch the latest version of each aspect in the local database and produce
|
||
MCL events corresponding to the aspects to restore the search and graph indices.</p><p>By default, restoring the indices from the local database will not remove any existing documents in
|
||
the search and graph indices that no longer exist in the local database, potentially leading to inconsistencies
|
||
between the search and graph indices and the local database.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuration">Configuration<a href="#configuration" class="hash-link" aria-label="Direct link to Configuration" title="Direct link to Configuration"></a></h2><p>The upgrade jobs take arguments as command line args to the job itself rather than environment variables for job specific
|
||
configuration. The RestoreIndices job is specified through the <code>-u RestoreIndices</code> upgrade ID parameter and then additional
|
||
parameters are specified like <code>-a batchSize=1000</code>.</p><p>The following configurations are available:</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="time-based-filtering">Time-Based Filtering<a href="#time-based-filtering" class="hash-link" aria-label="Direct link to Time-Based Filtering" title="Direct link to Time-Based Filtering"></a></h3><ul><li><code>lePitEpochMs</code>: Restore records created before this timestamp (in milliseconds)</li><li><code>gePitEpochMs</code>: Restore records created after this timestamp (in milliseconds)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pagination-and-performance-options">Pagination and Performance Options<a href="#pagination-and-performance-options" class="hash-link" aria-label="Direct link to Pagination and Performance Options" title="Direct link to Pagination and Performance Options"></a></h3><ul><li><code>urnBasedPagination</code>: Enable key-based pagination instead of offset-based pagination. Recommended for large datasets as it's typically more efficient.</li><li><code>startingOffset</code>: When using default pagination, start from this offset</li><li><code>lastUrn</code>: Resume from a specific URN when using URN-based pagination</li><li><code>lastAspect</code>: Used with lastUrn to resume from a specific aspect, preventing reprocessing</li><li><code>numThreads</code>: Number of concurrent threads for processing restoration, only used with default offset based paging</li><li><code>batchSize</code>: Configures the size of each batch as the job pages through rows</li><li><code>batchDelayMs</code>: Adds a delay in between each batch to avoid overloading backend systems</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="content-filtering">Content Filtering<a href="#content-filtering" class="hash-link" aria-label="Direct link to Content Filtering" title="Direct link to Content Filtering"></a></h3><ul><li><code>aspectNames</code>: Comma-separated list of aspects to restore (e.g., "ownership,status")</li><li><code>urnLike</code>: SQL LIKE pattern to filter URNs (e.g., "urn:li:dataset%")</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="default-aspects">Default Aspects<a href="#default-aspects" class="hash-link" aria-label="Direct link to Default Aspects" title="Direct link to Default Aspects"></a></h3><ul><li><code>createDefaultAspects</code>: Create default aspects in both SQL and ES if missing.</li></ul><p>During the restore indices process, it will create default aspects in SQL. While this may be
|
||
desired in some situations, disabling this feature is required when using a read-only SQL replica.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="nuclear-option">Nuclear option<a href="#nuclear-option" class="hash-link" aria-label="Direct link to Nuclear option" title="Direct link to Nuclear option"></a></h3><ul><li><code>clean</code>: This option wipes out the current indices by running deletes of all the documents to guarantee a consistent state with SQL. This is generally not recommended unless there is significant data corruption on the instance.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="helm">Helm<a href="#helm" class="hash-link" aria-label="Direct link to Helm" title="Direct link to Helm"></a></h3><p>These are available in the helm charts as configurations for Kubernetes deployments under the <code>datahubUpgrade.restoreIndices.args</code> path which will set them up as args to the pod command.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="execution-methods">Execution Methods<a href="#execution-methods" class="hash-link" aria-label="Direct link to Execution Methods" title="Direct link to Execution Methods"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="quickstart">Quickstart<a href="#quickstart" class="hash-link" aria-label="Direct link to Quickstart" title="Direct link to Quickstart"></a></h3><p>If you're using the quickstart images, you can use the <code>datahub</code> cli to restore the indices.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub </span><span class="token function" style="color:rgb(130, 170, 255)">docker</span><span class="token plain"> quickstart --restore-indices</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p>Using the <code>datahub</code> CLI to restore the indices when using the quickstart images will also clear the search and graph indices before restoring.</p></div></div><p>See <a href="/docs/1.1.0/quickstart#restore-datahub">this section</a> for more information.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="docker-compose">Docker-compose<a href="#docker-compose" class="hash-link" aria-label="Direct link to Docker-compose" title="Direct link to Docker-compose"></a></h3><p>If you are on a custom docker-compose deployment, run the following command (you need to checkout <a href="https://github.com/datahub-project/datahub" target="_blank" rel="noopener noreferrer">the source repository</a>) from the root of the repo to send MAE for each aspect in the local database.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">./docker/datahub-upgrade/datahub-upgrade.sh -u RestoreIndices</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p>By default this command will not clear the search and graph indices before restoring, thous potentially leading to inconsistencies between the local database and the indices, in case aspects were previously deleted in the local database but were not removed from the correponding index.</p></div></div><p>If you need to clear the search and graph indices before restoring, add <code>-a clean</code> to the end of the command. Please take note that the search and graph services might not be fully functional during reindexing when the indices are cleared.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">./docker/datahub-upgrade/datahub-upgrade.sh -u RestoreIndices -a clean</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Refer to this <a href="/docs/1.1.0/docker/datahub-upgrade#environment-variables">doc</a> on how to set environment variables
|
||
for your environment.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="kubernetes">Kubernetes<a href="#kubernetes" class="hash-link" aria-label="Direct link to Kubernetes" title="Direct link to Kubernetes"></a></h3><p>Run <code>kubectl get cronjobs</code> to see if the restoration job template has been deployed. If you see results like below, you
|
||
are good to go.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub-datahub-cleanup-job-template * * * * * True 0 <none> 2d3h</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">datahub-datahub-restore-indices-job-template * * * * * True 0 <none> 2d3h</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>If not, deploy latest helm charts to use this functionality.</p><p>Once restore indices job template has been deployed, run the following command to start a job that restores indices.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">kubectl create job --from</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">cronjob/datahub-datahub-restore-indices-job-template datahub-restore-indices-adhoc</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once the job completes, your indices will have been restored.</p><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p>By default the restore indices job template will not clear the search and graph indices before restoring, thous potentially leading to inconsistencies between the local database and the indices, in case aspects were previously deleted in the local database but were not removed from the correponding index.</p></div></div><p>If you need to clear the search and graph indices before restoring, modify the <code>values.yaml</code> for your deployment and overwrite the default arguments of the restore indices job template to include the <code>-a clean</code> argument. Please take note that the search and graph services might not be fully functional during reindexing when the indices are cleared.</p><div class="language-yaml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-yaml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token key atrule">datahubUpgrade</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">restoreIndices</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token key atrule">args</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"-u"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"RestoreIndices"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"-a"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"batchSize=1000"</span><span class="token plain"> </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># default value of datahubUpgrade.batchSize</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"-a"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"batchDelayMs=100"</span><span class="token plain"> </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic"># default value of datahubUpgrade.batchDelayMs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"-a"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"clean"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="through-apis">Through APIs<a href="#through-apis" class="hash-link" aria-label="Direct link to Through APIs" title="Direct link to Through APIs"></a></h3><p>See also the <a href="#best-practices">Best Practices</a> section below, however note that the APIs are able to handle a few thousand
|
||
aspects. In this mode one of the GMS instances will perform the required actions, however it is subject to timeout. Use one of the
|
||
approaches above for longer running restoreIndices.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="openapi">OpenAPI<a href="#openapi" class="hash-link" aria-label="Direct link to OpenAPI" title="Direct link to OpenAPI"></a></h4><p>There are two primary APIs, one which exposes the common parameters for restoreIndices and another one designed
|
||
to accept a list of URNs where all aspects are to be restored.</p><p>Full configuration:</p><p align="center"><img loading="lazy" width="80%" src="https://github.com/datahub-project/static-assets/blob/main/imgs/how/restore-indices/openapi-restore-indices.png?raw=true" class="img_ev3q"></p><p>All Aspects:</p><p align="center"><img loading="lazy" width="80%" src="https://github.com/datahub-project/static-assets/blob/main/imgs/how/restore-indices/openapi-restore-indices-urns.png?raw=true" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="restli">Rest.li<a href="#restli" class="hash-link" aria-label="Direct link to Rest.li" title="Direct link to Rest.li"></a></h4><p>For Rest.li, see <a href="/docs/1.1.0/api/restli/restore-indices">Restore Indices API</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="best-practices">Best Practices<a href="#best-practices" class="hash-link" aria-label="Direct link to Best Practices" title="Direct link to Best Practices"></a></h2><p>In general, this process is not required to run unless there has been a disruption of storage services or infrastructure,
|
||
such as Elasticsearch/Opensearch cluster failures, data corruption events, or significant version upgrade inconsistencies
|
||
that have caused the search and graph indices to become out of sync with the local database.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="k8-job-vs-api">K8 Job vs. API<a href="#k8-job-vs-api" class="hash-link" aria-label="Direct link to K8 Job vs. API" title="Direct link to K8 Job vs. API"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="when-to-use-kubernetes-jobs">When to Use Kubernetes Jobs<a href="#when-to-use-kubernetes-jobs" class="hash-link" aria-label="Direct link to When to Use Kubernetes Jobs" title="Direct link to When to Use Kubernetes Jobs"></a></h4><p>For operations affecting 2,000 or more aspects, it's strongly recommended to use the Kubernetes job approach. This job is
|
||
designed for long-running processes and provide several advantages:</p><ul><li>Won't time out like API calls</li><li>Can be monitored through Kubernetes logging</li><li>Won't consume resources from your primary GMS instances</li><li>Can be scheduled during off-peak hours to minimize system impact</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="when-to-use-apis">When to Use APIs<a href="#when-to-use-apis" class="hash-link" aria-label="Direct link to When to Use APIs" title="Direct link to When to Use APIs"></a></h4><p>The RestoreIndices APIs (available through both Rest.li and OpenAPI) is best suited for:</p><ul><li>Targeted restores affecting fewer than 2,000 aspects</li><li>Emergencies where you need to quickly restore critical metadata</li><li>Testing or validating the restore process before running a full-scale job</li><li>Scenarios where you don't have direct access to the Kubernetes cluster</li></ul><p>Remember that API-based restoration runs within one of your GMS instances and is subject to timeouts, which could lead to
|
||
incomplete restorations for larger installations.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="targeted-restoration-strategies">Targeted Restoration Strategies<a href="#targeted-restoration-strategies" class="hash-link" aria-label="Direct link to Targeted Restoration Strategies" title="Direct link to Targeted Restoration Strategies"></a></h3><p>Being selective about what you restore is crucial for efficiency. Combining these filtering strategies can dramatically
|
||
reduce the restoration scope, saving resources and time.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="entity-type-filtering">Entity Type Filtering<a href="#entity-type-filtering" class="hash-link" aria-label="Direct link to Entity Type Filtering" title="Direct link to Entity Type Filtering"></a></h4><p>Entity Type Filtering: Use the <code>urnLike</code> parameter to target specific entity types:</p><ul><li>For datasets: <code>urnLike=urn:li:dataset:%</code></li><li>For users: <code>urnLike=urn:li:corpuser:%</code></li><li>For dashboards: <code>urnLike=urn:li:dashboard:%</code></li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="single-entity">Single Entity<a href="#single-entity" class="hash-link" aria-label="Direct link to Single Entity" title="Direct link to Single Entity"></a></h4><p>Single Entity Restoration: When only one entity is affected, provide the specific URN to minimize processing overhead.
|
||
Aspect-Based Filtering: Use aspectNames to target only the specific aspects that need restoration:</p><ul><li>For ownership inconsistencies: <code>aspectNames=ownership</code></li><li>For tag issues: <code>aspectNames=globalTags</code></li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="time-based">Time-Based<a href="#time-based" class="hash-link" aria-label="Direct link to Time-Based" title="Direct link to Time-Based"></a></h4><p>Time-Based Recovery: If you know when the inconsistency began, use time-based filtering:</p><ul><li>gePitEpochMs={timestamp} to process only records created after the incident</li><li>lePitEpochMs={timestamp} to limit processing to records before a certain time</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="parallel-processing-strategies">Parallel Processing Strategies<a href="#parallel-processing-strategies" class="hash-link" aria-label="Direct link to Parallel Processing Strategies" title="Direct link to Parallel Processing Strategies"></a></h3><p>To optimize restoration speed while managing system load:</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="multiple-parallel-jobs">Multiple Parallel Jobs<a href="#multiple-parallel-jobs" class="hash-link" aria-label="Direct link to Multiple Parallel Jobs" title="Direct link to Multiple Parallel Jobs"></a></h4><p>Run several restoreIndices processes simultaneously by:</p><ul><li>Work on non-overlapping sets of aspects or entities<ul><li>Dividing work by entity type (one job for datasets, another for users, etc.)</li><li>Splitting aspects among different jobs (one for ownership aspects, another for lineage, etc.)</li><li>Partitioning large entity types by prefix or time range</li></ul></li><li>Have staggered start times to prevent initial resource contention</li><li>Monitor system metrics closely during concurrent restoration to ensure you're not overloading your infrastructure.</li></ul><div class="theme-admonition theme-admonition-caution alert alert--warning admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 16 16"><path fill-rule="evenodd" d="M8.893 1.5c-.183-.31-.52-.5-.887-.5s-.703.19-.886.5L.138 13.499a.98.98 0 0 0 0 1.001c.193.31.53.501.886.501h13.964c.367 0 .704-.19.877-.5a1.03 1.03 0 0 0 .01-1.002L8.893 1.5zm.133 11.497H6.987v-2.003h2.039v2.003zm0-3.004H6.987V5.987h2.039v4.006z"></path></svg></span>caution</div><div class="admonitionContent_S0QG"><p>Avoid Conflicts: Ensure that concurrent jobs:</p><p>Never specify the --clean argument in concurrent jobs</p></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="temporary-workload-reduction">Temporary Workload Reduction<a href="#temporary-workload-reduction" class="hash-link" aria-label="Direct link to Temporary Workload Reduction" title="Direct link to Temporary Workload Reduction"></a></h3><ul><li>Pause scheduled ingestion jobs during restoration</li><li>Temporarily disable or reduce frequency of the datahub-gc job to prevent conflicting deletes</li><li>Consider pausing automated workflows or integrations that generate metadata events</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="infrastructure-tuning">Infrastructure Tuning<a href="#infrastructure-tuning" class="hash-link" aria-label="Direct link to Infrastructure Tuning" title="Direct link to Infrastructure Tuning"></a></h3><p>Implementing these expanded best practices should help ensure a smoother, more efficient restoration process while
|
||
minimizing impact on your DataHub environment.</p><p>This operation can be I/O intensive from the read-side from SQL and on the Elasticsearch write side. If you're able to leverage
|
||
provisioned I/O. or throughput, you might want to monitor your infrastructure for a possible.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="elasticsearchopensearch-optimization">Elasticsearch/Opensearch Optimization<a href="#elasticsearchopensearch-optimization" class="hash-link" aria-label="Direct link to Elasticsearch/Opensearch Optimization" title="Direct link to Elasticsearch/Opensearch Optimization"></a></h4><p>To improve write performance during restoration:</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="refresh-interval-adjustment">Refresh Interval Adjustment:<a href="#refresh-interval-adjustment" class="hash-link" aria-label="Direct link to Refresh Interval Adjustment:" title="Direct link to Refresh Interval Adjustment:"></a></h5><p>Temporarily increase the refresh_interval setting from the default (typically 1s) to something like 30s or 60s.
|
||
Run the system update job with the following environment variable <code>ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=60</code></p><div class="theme-admonition theme-admonition-caution alert alert--warning admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 16 16"><path fill-rule="evenodd" d="M8.893 1.5c-.183-.31-.52-.5-.887-.5s-.703.19-.886.5L.138 13.499a.98.98 0 0 0 0 1.001c.193.31.53.501.886.501h13.964c.367 0 .704-.19.877-.5a1.03 1.03 0 0 0 .01-1.002L8.893 1.5zm.133 11.497H6.987v-2.003h2.039v2.003zm0-3.004H6.987V5.987h2.039v4.006z"></path></svg></span>caution</div><div class="admonitionContent_S0QG"><p>Remember to reset this after restoration completes!</p></div></div><h5 class="anchor anchorWithStickyNavbar_LWe7" id="bulk-processing-improvements">Bulk Processing Improvements:<a href="#bulk-processing-improvements" class="hash-link" aria-label="Direct link to Bulk Processing Improvements:" title="Direct link to Bulk Processing Improvements:"></a></h5><ul><li>Adjust the Elasticsearch batching parameters to optimize bulk request size (try values between 2000-5000)<ul><li>Run your GMS or <code>mae-consumer</code> with environment variables<ul><li><code>ES_BULK_REQUESTS_LIMIT=3000</code></li><li><code>ES_BULK_FLUSH_PERIOD=60</code></li></ul></li></ul></li><li>Configure <code>batchDelayMs</code> on restoreIndices to add breathing room between batches if your cluster is struggling</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="shard-management">Shard Management:<a href="#shard-management" class="hash-link" aria-label="Direct link to Shard Management:" title="Direct link to Shard Management:"></a></h5><ul><li>Ensure your indices have an appropriate number of shards for your cluster size.</li><li>Consider temporarily adding nodes to your search cluster during massive restorations.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="sqlprimary-storage">SQL/Primary Storage<a href="#sqlprimary-storage" class="hash-link" aria-label="Direct link to SQL/Primary Storage" title="Direct link to SQL/Primary Storage"></a></h4><p>Consider using a read replica as the source of the job's data. If you configure a read-only replica
|
||
you must also provide the parameter <code>createDefaultAspects=false</code>.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="kafka--consumers">Kafka & Consumers<a href="#kafka--consumers" class="hash-link" aria-label="Direct link to Kafka & Consumers" title="Direct link to Kafka & Consumers"></a></h4><h5 class="anchor anchorWithStickyNavbar_LWe7" id="partition-strategy">Partition Strategy:<a href="#partition-strategy" class="hash-link" aria-label="Direct link to Partition Strategy:" title="Direct link to Partition Strategy:"></a></h5><ul><li>Verify that the Kafka Metadata Change Log (MCL) topic have enough partitions to allow for parallel processing.</li><li>Recommended: At least 10-20 partitions for the MCL topic in production environments.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="consumer-scaling">Consumer Scaling:<a href="#consumer-scaling" class="hash-link" aria-label="Direct link to Consumer Scaling:" title="Direct link to Consumer Scaling:"></a></h5><ul><li>Temporarily increase the number of <code>mae-consumer</code> pods to process the higher event volume.</li><li>Scale GMS instances if they're handling consumer duties.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="monitoring">Monitoring:<a href="#monitoring" class="hash-link" aria-label="Direct link to Monitoring:" title="Direct link to Monitoring:"></a></h5><ul><li>Watch consumer lag metrics closely during restoration.</li><li>Be prepared to adjust scaling or batch parameters if consumers fall behind.</li></ul></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="slackUtm_uoBr"><div class="slackUtm_uoBr"><hr>Need more help? Join the conversation in <a href="https://datahub.com/slack?utm_source=docs&utm_medium=footer&utm_campaign=docs_footer&utm_content=docs/how/restore-indices">Slack!</a></div></div><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/datahub-project/datahub/blob/master/docs/how/restore-indices.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_VsjB"></div></div></footer><div class="feedbackWrapper_mUHF"><div class="feedbackWidget_PX4d"><div class="feedbackButtons_wn3V"><strong>Is this page helpful?</strong><div><button class="feedbackButton_UgQs"><span role="img" aria-label="like" class="anticon anticon-like"><svg viewBox="64 64 896 896" focusable="false" data-icon="like" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M885.9 533.7c16.8-22.2 26.1-49.4 26.1-77.7 0-44.9-25.1-87.4-65.5-111.1a67.67 67.67 0 00-34.3-9.3H572.4l6-122.9c1.4-29.7-9.1-57.9-29.5-79.4A106.62 106.62 0 00471 99.9c-52 0-98 35-111.8 85.1l-85.9 311H144c-17.7 0-32 14.3-32 32v364c0 17.7 14.3 32 32 32h601.3c9.2 0 18.2-1.8 26.5-5.4 47.6-20.3 78.3-66.8 78.3-118.4 0-12.6-1.8-25-5.4-37 16.8-22.2 26.1-49.4 26.1-77.7 0-12.6-1.8-25-5.4-37 16.8-22.2 26.1-49.4 26.1-77.7-.2-12.6-2-25.1-5.6-37.1zM184 852V568h81v284h-81zm636.4-353l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 16.5-7.2 32.2-19.6 43l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 16.5-7.2 32.2-19.6 43l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 22.4-13.2 42.6-33.6 51.8H329V564.8l99.5-360.5a44.1 44.1 0 0142.2-32.3c7.6 0 15.1 2.2 21.1 6.7 9.9 7.4 15.2 18.6 14.6 30.5l-9.6 198.4h314.4C829 418.5 840 436.9 840 456c0 16.5-7.2 32.1-19.6 43z"></path></svg></span></button><button class="feedbackButton_UgQs"><span role="img" aria-label="dislike" class="anticon anticon-dislike"><svg viewBox="64 64 896 896" focusable="false" data-icon="dislike" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M885.9 490.3c3.6-12 5.4-24.4 5.4-37 0-28.3-9.3-55.5-26.1-77.7 3.6-12 5.4-24.4 5.4-37 0-28.3-9.3-55.5-26.1-77.7 3.6-12 5.4-24.4 5.4-37 0-51.6-30.7-98.1-78.3-118.4a66.1 66.1 0 00-26.5-5.4H144c-17.7 0-32 14.3-32 32v364c0 17.7 14.3 32 32 32h129.3l85.8 310.8C372.9 889 418.9 924 470.9 924c29.7 0 57.4-11.8 77.9-33.4 20.5-21.5 31-49.7 29.5-79.4l-6-122.9h239.9c12.1 0 23.9-3.2 34.3-9.3 40.4-23.5 65.5-66.1 65.5-111 0-28.3-9.3-55.5-26.1-77.7zM184 456V172h81v284h-81zm627.2 160.4H496.8l9.6 198.4c.6 11.9-4.7 23.1-14.6 30.5-6.1 4.5-13.6 6.8-21.1 6.7a44.28 44.28 0 01-42.2-32.3L329 459.2V172h415.4a56.85 56.85 0 0133.6 51.8c0 9.7-2.3 18.9-6.9 27.3l-13.9 25.4 21.9 19a56.76 56.76 0 0119.6 43c0 9.7-2.3 18.9-6.9 27.3l-13.9 25.4 21.9 19a56.76 56.76 0 0119.6 43c0 9.7-2.3 18.9-6.9 27.3l-14 25.5 21.9 19a56.76 56.76 0 0119.6 43c0 19.1-11 37.5-28.8 48.4z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/1.1.0/how/backup-datahub"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Taking backup of DataHub</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/1.1.0/advanced/db-retention"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Configuring Database Retention</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link toc-highlight">Configuration</a><ul><li><a href="#time-based-filtering" class="table-of-contents__link toc-highlight">Time-Based Filtering</a></li><li><a href="#pagination-and-performance-options" class="table-of-contents__link toc-highlight">Pagination and Performance Options</a></li><li><a href="#content-filtering" class="table-of-contents__link toc-highlight">Content Filtering</a></li><li><a href="#default-aspects" class="table-of-contents__link toc-highlight">Default Aspects</a></li><li><a href="#nuclear-option" class="table-of-contents__link toc-highlight">Nuclear option</a></li><li><a href="#helm" class="table-of-contents__link toc-highlight">Helm</a></li></ul></li><li><a href="#execution-methods" class="table-of-contents__link toc-highlight">Execution Methods</a><ul><li><a href="#quickstart" class="table-of-contents__link toc-highlight">Quickstart</a></li><li><a href="#docker-compose" class="table-of-contents__link toc-highlight">Docker-compose</a></li><li><a href="#kubernetes" class="table-of-contents__link toc-highlight">Kubernetes</a></li><li><a href="#through-apis" class="table-of-contents__link toc-highlight">Through APIs</a></li></ul></li><li><a href="#best-practices" class="table-of-contents__link toc-highlight">Best Practices</a><ul><li><a href="#k8-job-vs-api" class="table-of-contents__link toc-highlight">K8 Job vs. API</a></li><li><a href="#targeted-restoration-strategies" class="table-of-contents__link toc-highlight">Targeted Restoration Strategies</a></li><li><a href="#parallel-processing-strategies" class="table-of-contents__link toc-highlight">Parallel Processing Strategies</a></li><li><a href="#temporary-workload-reduction" class="table-of-contents__link toc-highlight">Temporary Workload Reduction</a></li><li><a href="#infrastructure-tuning" class="table-of-contents__link toc-highlight">Infrastructure Tuning</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/docs/">Introduction</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/quickstart">Quickstart</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://datahub.com/slack" target="_blank" rel="noopener noreferrer" class="footer__link-item">Slack<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="footer__link-item">YouTube<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="footer__link-item">Blog<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/docs/townhalls">Town Halls</a></li><li class="footer__item"><a href="https://datahub.com/adoption-stories/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Adoption<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://demo.datahub.com/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Demo</a></li><li class="footer__item"><a href="https://feature-requests.datahubproject.io/roadmap" target="_blank" rel="noopener noreferrer" class="footer__link-item">Roadmap<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/docs/contributing">Contributing</a></li><li class="footer__item"><a href="https://github.com/datahub-project/datahub" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://feature-requests.datahubproject.io/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Feature Requests<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2015-2025 DataHub Project Authors.</div></div></div></footer></div>
|
||
<script src="/assets/js/runtime~main.d9d0eca5.js"></script>
|
||
<script src="/assets/js/main.75f9b227.js"></script>
|
||
</body>
|
||
</html> |