mirror of
https://github.com/datahub-project/datahub.git
synced 2025-07-29 20:39:56 +00:00
173 lines
108 KiB
HTML
173 lines
108 KiB
HTML
<!doctype html>
|
||
<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-docs/managed-datahub/observe/freshness-assertions" data-has-hydrated="false">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="generator" content="Docusaurus v2.4.3">
|
||
<title data-rh="true">Freshness Assertions | DataHub</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://docs.datahub.com/docs/managed-datahub/observe/freshness-assertions"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Freshness Assertions | DataHub"><meta data-rh="true" name="description" content="This page provides an overview of working with DataHub Freshness Assertions"><meta data-rh="true" property="og:description" content="This page provides an overview of working with DataHub Freshness Assertions"><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://docs.datahub.com/docs/managed-datahub/observe/freshness-assertions"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/managed-datahub/observe/freshness-assertions" hreflang="en"><link data-rh="true" rel="alternate" href="https://docs.datahub.com/docs/managed-datahub/observe/freshness-assertions" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://RK0UG797F3-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/learn/rss.xml" title="DataHub RSS Feed">
|
||
<link rel="alternate" type="application/atom+xml" href="/learn/atom.xml" title="DataHub Atom Feed">
|
||
|
||
<link rel="preconnect" href="https://www.google-analytics.com">
|
||
<link rel="preconnect" href="https://www.googletagmanager.com">
|
||
<script async src="https://www.googletagmanager.com/gtag/js?id=G-PKGVLETT4C"></script>
|
||
<script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-PKGVLETT4C",{})</script>
|
||
<link rel="preconnect" href="https://www.googletagmanager.com">
|
||
<script>window.dataLayer=window.dataLayer||[]</script>
|
||
<script>!function(e,t,a,n,g){e[n]=e[n]||[],e[n].push({"gtm.start":(new Date).getTime(),event:"gtm.js"});var m=t.getElementsByTagName(a)[0],r=t.createElement(a);r.async=!0,r.src="https://www.googletagmanager.com/gtm.js?id=GTM-5M8T9HNN",m.parentNode.insertBefore(r,m)}(window,document,"script","dataLayer")</script>
|
||
|
||
|
||
<link rel="search" type="application/opensearchdescription+xml" title="DataHub" href="/opensearch.xml">
|
||
|
||
|
||
|
||
|
||
<meta httpequiv="Content-Security-Policy" content="frame-ancestors 'self' https://*.acryl.io https://acryldata.io http://localhost:*">
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap">
|
||
<script src="https://tools.luckyorange.com/core/lo.js?site-id=28ea8a38" async defer="defer"></script>
|
||
<script src="/scripts/rb2b.js" async defer="defer"></script>
|
||
<script src="https://app.revenuehero.io/scheduler.min.js"></script>
|
||
<script src="https://tag.clearbitscripts.com/v1/pk_2e321cabe30432a5c44c0424781aa35f/tags.js" referrerpolicy="strict-origin-when-cross-origin"></script>
|
||
<script src="/scripts/reo.js"></script>
|
||
<script id="runllm-widget-script" type="module" src="https://widget.runllm.com" crossorigin="true" runllm-name="DataHub" runllm-assistant-id="81" runllm-position="BOTTOM_RIGHT" runllm-keyboard-shortcut="Mod+j" runllm-preset="docusaurus" runllm-theme-color="#1890FF" runllm-brand-logo="https://docs.datahub.com/img/datahub-logo-color-mark.svg" runllm-community-url="https://datahub.com/slack" runllm-community-type="slack" runllm-disable-ask-a-person="true" async></script><link rel="stylesheet" href="/assets/css/styles.d8fe2eb8.css">
|
||
<link rel="preload" href="/assets/js/runtime~main.8ae4198a.js" as="script">
|
||
<link rel="preload" href="/assets/js/main.9d79f7e2.js" as="script">
|
||
</head>
|
||
<body class="navigation-with-keyboard">
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-5M8T9HNN" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
|
||
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
|
||
<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:transparent;color:#ffffff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"><div class="shimmer-banner"><p>DataHub Secures $35 Million Series B</p><a href="https://datahub.com/news/series-b-announcement/" target="_blank" class="button"><div>Read the announcement<span> →</span></div></a></div></div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a href="https://datahub.com" target="_blank" rel="noopener noreferrer" class="navbar__brand"><div class="navbar__logo"><img src="/img/datahub-logo-color-light-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/datahub-logo-color-dark-horizontal.svg" alt="DataHub Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link versionNavItem_cbn8">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/managed-datahub/observe/freshness-assertions">Next</a></li><li><a class="dropdown__link" href="/docs/1.1.0/managed-datahub/observe/freshness-assertions">1.1.0</a></li><li><hr class="dropdown-separator" style="margin: 0.4rem;"></li><li><div class="dropdown__link"><b>Archived versions</b></div></li><li>
|
||
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/features">1.0.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-t9sv4w3gr-acryldata.vercel.app/docs/0.15.0/features">0.15.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-8jkm4uler-acryldata.vercel.app/docs/0.14.1/features">0.14.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-eue2qafvn-acryldata.vercel.app/docs/features">0.14.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-psat3nzgi-acryldata.vercel.app/docs/features">0.13.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-lzxh86531-acryldata.vercel.app/docs/features">0.13.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-2uuxmgza2-acryldata.vercel.app/docs/features">0.12.1
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-irpoe2osc-acryldata.vercel.app/docs/features">0.11.0
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li><li>
|
||
<a class="dropdown__link" href="https://docs-website-1gv2yzn9d-acryldata.vercel.app/docs/features">0.10.5
|
||
<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
|
||
</a>
|
||
</li></ul></div></div><div class="navbar__items navbar__items--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/docs">Docs</a><a class="navbar__item navbar__link" href="/integrations">Integrations</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/weekly-demo" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Weekly Demo"></div><div class="title_c7DP">Weekly Demo</div></a></div><div><a href="https://datahub.com/use-cases" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-forum.png" alt="Use Cases"></div><div class="title_c7DP">Use Cases</div></a></div><div><a href="httpps://datahub.com/adoption-stories" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Adoption Stories"></div><div class="title_c7DP">Adoption Stories</div></a></div><div><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Blog"></div><div class="title_c7DP">Blog</div></a></div><div><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Youtube"></div><div class="title_c7DP">Youtube</div></a></div></div></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu dropdown__menu_Z8FC"><div class="wrapper_kp81"><div><a href="https://datahub.com/slack/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-join-slack.png" alt="Join Slack"></div><div class="title_c7DP">Join Slack</div></a></div><div><a href="https://datahub.com/events" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-events.png" alt="Events"></div><div class="title_c7DP">Events</div></a></div><div><a href="https://datahub.com/champions/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-champions.png" alt="Champions"></div><div class="title_c7DP">Champions</div></a></div><div><a href="https://datahub.com/share-your-journey/" target="_blank" rel="noopener noreferrer" class="card_BUD7"><div class="icon_BgHd"><img src="/img/icon-share-your-journey.png" alt="Share Your Journey"></div><div class="title_c7DP">Share Your Journey</div></a></div></div></ul></div><a href="https://datahub.com/products/why-datahub-cloud/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
||
<style>
|
||
.cloud-cta {
|
||
color: var(--ifm-menu-color-active);
|
||
font-weight: 600;
|
||
background: linear-gradient(40deg, var(--ifm-menu-color-active), var(--ifm-menu-color-active));
|
||
background-size: 200% 100%;
|
||
-webkit-background-clip: text;
|
||
background-clip: text;
|
||
transition: background-image 0.3s ease;
|
||
}
|
||
.cloud-cta:hover {
|
||
color: transparent;
|
||
background: linear-gradient(40deg, var(--ifm-menu-color-active), #ff1493);
|
||
background-size: 200% 100%;
|
||
-webkit-background-clip: text;
|
||
background-clip: text;
|
||
animation: gradientShift 3s ease infinite;
|
||
}
|
||
@keyframes gradientShift {
|
||
0%, 100% { background-position: 0% 50%; }
|
||
50% { background-position: 100% 50%; }
|
||
}
|
||
</style>
|
||
<div class="cloud-cta">Get Cloud</div>
|
||
</a><a href="https://datahub.com/slack?utm_source=docs&utm_medium=header&utm_campaign=docs_header" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">
|
||
<style>
|
||
.slack-logo:hover {
|
||
opacity: 0.8;
|
||
}
|
||
</style>
|
||
<img class="slack-logo" src="https://upload.wikimedia.org/wikipedia/commons/d/d5/Slack_icon_2019.svg" , alt="slack" , height="20px" style="margin: 10px 0 0 0;">
|
||
</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Getting Started</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/features">What Is DataHub?</a><button aria-label="Toggle the collapsible sidebar category 'What Is DataHub?'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--active" aria-expanded="true" href="/docs/category/features">Features</a><button aria-label="Toggle the collapsible sidebar category 'Features'" type="button" class="clean-btn menu__caret"></button></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--active" aria-expanded="true" tabindex="0" href="/docs/managed-datahub/observe/assertions">Assertions (Data Quality)</a><button aria-label="Toggle the collapsible sidebar category 'Assertions (Data Quality)'" type="button" class="clean-btn menu__caret"></button></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item saasOnly"><a class="menu__link" tabindex="0" href="/docs/managed-datahub/observe/column-assertions">Column Assertions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item saasOnly"><a class="menu__link" tabindex="0" href="/docs/managed-datahub/observe/custom-sql-assertions">Custom SQL Assertions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item saasOnly"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/managed-datahub/observe/freshness-assertions">Freshness Assertions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item saasOnly"><a class="menu__link" tabindex="0" href="/docs/managed-datahub/observe/schema-assertions">Schema Assertions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item saasOnly"><a class="menu__link" tabindex="0" href="/docs/managed-datahub/observe/volume-assertions">Volume Assertions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item saasOnly"><a class="menu__link" tabindex="0" href="/docs/managed-datahub/observe/smart-assertions">Smart Assertions ⚡ (Anomaly Detection)</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item saasOnly"><a class="menu__link" tabindex="0" href="/docs/managed-datahub/observe/data-health-dashboard">Data Health Dashboard</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-3 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" tabindex="0" href="/docs/assertions/open-assertions-spec">Open Assertions Specification</a><button aria-label="Toggle the collapsible sidebar category 'Open Assertions Specification'" type="button" class="clean-btn menu__caret"></button></div></li></ul></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/features/feature-guides/access-management">Access Management</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/automations/docs-propagation">Automations</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/businessattributes">Business Attributes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/glossary/business-glossary">Business Glossary</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/features/feature-guides/compliance-forms/overview">Compliance Forms</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/managed-datahub/observe/data-contract">Data Contract</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/dataproducts">Data Products</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/features/dataset-usage-and-query-history">Dataset Usage and Query History</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/domains">Domains</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/incidents/incidents">Incidents</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/ui-ingestion">Ingestion</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" tabindex="0" href="/docs/features/feature-guides/lineage">Lineage</a><button aria-label="Toggle the collapsible sidebar category 'Lineage'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item saasOnly"><a class="menu__link" tabindex="0" href="/docs/tests/metadata-tests">Metadata Tests</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/features/feature-guides/mcp">MCP Server</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/ownership/ownership-types">Ownership</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/authorization/access-policies-guide">Policies</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/posts">Posts</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/features/feature-guides/properties/overview">Properties</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/schema-history">Schema history</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/how/search">Search</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/sync-status">Sync Status</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/tags">Tags</a></li></ul></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>DataHub Cloud</div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/managed-datahub-overview">DataHub Cloud Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/welcome-acryl">Getting Started with DataHub Cloud</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/upgrade_core_to_cloud">Upgrading from DataHub Core to Cloud</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authentication/guides/sso/initialize-oidc">Configure Single Sign-On</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/remote-executor/about">Remote Executor</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/datahub-api/entity-events-api">DataHub API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/slack/saas-slack-app">Slack</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge">Operator Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item saasOnly"><a class="menu__link" href="/docs/managed-datahub/approval-workflows">Change Proposals & Approval Workflows</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/managed-datahub/chrome-extension">Cloud Chrome Extension</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item saasOnly"><a class="menu__link" href="/docs/managed-datahub/subscription-and-notification">Subscriptions & Notifications</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/managed-datahub/release-notes/v_0_3_12">DataHub Cloud Release History</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Integrations</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-ingestion">Overview</a><button aria-label="Toggle the collapsible sidebar category 'Overview'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/cli-ingestion">Quickstart Guides</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-ingestion/source_overview">Sources</a><button aria-label="Toggle the collapsible sidebar category 'Sources'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/schedule_docs/intro">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>API & SDKs</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-modeling/metadata-model">DataHub's Open Metadata Standard</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/what-is-datahub/datahub-concepts">Concepts</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/metadata-standards">Metadata Standards</a><button aria-label="Toggle the collapsible sidebar category 'Metadata Standards'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/api/datahub-apis">APIs and SDKs Overview</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/api/graphql/overview">API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/metadata-ingestion/as-a-library">Python SDK</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/metadata-integration/java/as-a-library">Java SDK</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/cli">DataHub CLI</a><button aria-label="Toggle the collapsible sidebar category 'DataHub CLI'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/act-on-metadata">DataHub Actions</a><button aria-label="Toggle the collapsible sidebar category 'DataHub Actions'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/api/tutorials/datasets">Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Admin</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authentication">Authentication</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/authorization">Authorization</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/how/delete-metadata">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Deployment</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/deployment-guides">Deployment Guides</a><button aria-label="Toggle the collapsible sidebar category 'Deployment Guides'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/deploy/confluent-cloud">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Developers</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/architecture/architecture">Architecture</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/developers">Developing on DataHub</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/advanced/mcp-mcl">Advanced Guides</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menuHtmlItem_M9Kj menu__list-item"><div>Community</div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/category/community">Community</a><button aria-label="Toggle the collapsible sidebar category 'Community'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/releases">Release History</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item"><a class="breadcrumbs__link" itemprop="item" href="/docs/category/features"><span itemprop="name">Features</span></a><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item"><a class="breadcrumbs__link" itemprop="item" href="/docs/managed-datahub/observe/assertions"><span itemprop="name">Assertions (Data Quality)</span></a><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Freshness Assertions</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Freshness Assertions</h1><div class="availabilityCard_P5od card"><strong>Feature Availability</strong><div><span class="platform_wqXv">Self-Hosted DataHub <span role="img" aria-label="close-circle" class="anticon anticon-close-circle"><svg fill-rule="evenodd" viewBox="64 64 896 896" focusable="false" data-icon="close-circle" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M512 64c247.4 0 448 200.6 448 448S759.4 960 512 960 64 759.4 64 512 264.6 64 512 64zm127.98 274.82h-.04l-.08.06L512 466.75 384.14 338.88c-.04-.05-.06-.06-.08-.06a.12.12 0 00-.07 0c-.03 0-.05.01-.09.05l-45.02 45.02a.2.2 0 00-.05.09.12.12 0 000 .07v.02a.27.27 0 00.06.06L466.75 512 338.88 639.86c-.05.04-.06.06-.06.08a.12.12 0 000 .07c0 .03.01.05.05.09l45.02 45.02a.2.2 0 00.09.05.12.12 0 00.07 0c.02 0 .04-.01.08-.05L512 557.25l127.86 127.87c.04.04.06.05.08.05a.12.12 0 00.07 0c.03 0 .05-.01.09-.05l45.02-45.02a.2.2 0 00.05-.09.12.12 0 000-.07v-.02a.27.27 0 00-.05-.06L557.25 512l127.87-127.86c.04-.04.05-.06.05-.08a.12.12 0 000-.07c0-.03-.01-.05-.05-.09l-45.02-45.02a.2.2 0 00-.09-.05.12.12 0 00-.07 0z"></path></svg></span></span></div><div><span role="img" aria-label="cloud" class="anticon anticon-cloud managedIcon_AxXO"><svg viewBox="64 64 896 896" focusable="false" data-icon="cloud" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M811.4 418.7C765.6 297.9 648.9 212 512.2 212S258.8 297.8 213 418.6C127.3 441.1 64 519.1 64 612c0 110.5 89.5 200 199.9 200h496.2C870.5 812 960 722.5 960 612c0-92.7-63.1-170.7-148.6-193.3zm36.3 281a123.07 123.07 0 01-87.6 36.3H263.9c-33.1 0-64.2-12.9-87.6-36.3A123.3 123.3 0 01140 612c0-28 9.1-54.3 26.2-76.3a125.7 125.7 0 0166.1-43.7l37.9-9.9 13.9-36.6c8.6-22.8 20.6-44.1 35.7-63.4a245.6 245.6 0 0152.4-49.9c41.1-28.9 89.5-44.2 140-44.2s98.9 15.3 140 44.2c19.9 14 37.5 30.8 52.4 49.9 15.1 19.3 27.1 40.7 35.7 63.4l13.8 36.5 37.8 10c54.3 14.5 92.1 63.8 92.1 120 0 33.1-12.9 64.3-36.3 87.7z"></path></svg></span><span class="platform_wqXv platformAvailable_Y8lN">DataHub Cloud <span role="img" aria-label="check-circle" class="anticon anticon-check-circle"><svg viewBox="64 64 896 896" focusable="false" data-icon="check-circle" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm193.5 301.7l-210.6 292a31.8 31.8 0 01-51.7 0L318.5 484.9c-3.8-5.3 0-12.7 6.5-12.7h46.9c10.2 0 19.9 4.9 25.9 13.3l71.2 98.8 157.2-218c6-8.3 15.6-13.3 25.9-13.3H699c6.5 0 10.3 7.4 6.5 12.7z"></path></svg></span></span></div></div><blockquote><p>The <strong>Freshness Assertions</strong> feature is available as part of the <strong>DataHub Cloud Observe</strong> module of DataHub Cloud.
|
||
If you are interested in learning more about <strong>DataHub Cloud Observe</strong> or trying it out, please <a href="https://datahub.com/products/data-observability/" target="_blank" rel="noopener noreferrer">visit our website</a>.</p></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="introduction">Introduction<a href="#introduction" class="hash-link" aria-label="Direct link to Introduction" title="Direct link to Introduction"></a></h2><p>Can you remember a time when a Data Warehouse Table that you depended on went days, weeks, or even
|
||
months without being updated with fresh data?</p><p>Perhaps a bug had been introduced into an upstream Airflow DAG
|
||
or worse, the person in charge of maintaining the Table has departed from your organization entirely.
|
||
There are many reasons why an important Table on Snowflake, Redshift, BigQuery, or Databricks may fail to be updated as often as expected.</p><p>What if you could reduce the time to detect these incidents, so that the people responsible for the data were made aware of data
|
||
issues <em>before</em> anyone else? What if you could communicate commitments about the freshness or change frequency
|
||
of a table? With DataHub Cloud Freshness Assertions, you can.</p><p>DataHub Cloud allows users to define expectations about when a particular Table in the warehouse
|
||
should change, and then monitor those expectations over time, with the ability to be notified when things go wrong.</p><p>In this article, we'll cover the basics of monitoring Freshness Assertions - what they are, how to configure them, and more - so that you and your team can
|
||
start building trust in your most important data assets.</p><p>Let's get started!</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="support">Support<a href="#support" class="hash-link" aria-label="Direct link to Support" title="Direct link to Support"></a></h2><p>Freshness Assertions are currently supported for:</p><ol><li>Snowflake</li><li>Redshift</li><li>BigQuery</li><li>Databricks</li><li>DataHub Operations (collected via ingestion)</li></ol><p>Note that an Ingestion Source <em>must</em> be configured with the data platform of your choice in DataHub Cloud's <strong>Ingestion</strong>
|
||
tab.</p><blockquote><p>Note that Freshness Assertions are not yet supported if you are connecting to your warehouse
|
||
using the DataHub CLI.</p></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="what-is-a-freshness-assertion">What is a Freshness Assertion?<a href="#what-is-a-freshness-assertion" class="hash-link" aria-label="Direct link to What is a Freshness Assertion?" title="Direct link to What is a Freshness Assertion?"></a></h2><p>A <strong>Freshness Assertion</strong> is a configurable Data Quality rule used to determine whether a Table
|
||
on the Data Warehouse has been updated within a given period of time. Freshness Assertions are particularly useful when you have frequently-changing
|
||
Tables.</p><p>For example, imagine that we work for a company with a Snowflake Table that stores user clicks collected from our e-commerce website.
|
||
This table is updated with new data on a specific cadence: once per hour (In practice, daily or even weekly are also common).
|
||
In turn, there is a downstream Business Analytics Dashboard in Looker that shows important metrics like
|
||
the number of people clicking our "Daily Sale" banners, and this dashboard is generated from data stored in our "clicks" table.
|
||
It is important that our clicks Table continues to be updated each hour because if it stops being updated, it could mean
|
||
that our downstream metrics dashboard becomes incorrect. And the risk of this situation is obvious: our organization
|
||
may make bad decisions based on incomplete information.</p><p>In such cases, we can use a <strong>Freshness Assertion</strong> that checks whether the Snowflake "clicks" Table is being updated with
|
||
fresh data each and every hour as expected. If an hour goes by without any changes, we can immediately notify our team, to prevent any
|
||
negative impacts.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="anatomy-of-a-freshness-assertion">Anatomy of a Freshness Assertion<a href="#anatomy-of-a-freshness-assertion" class="hash-link" aria-label="Direct link to Anatomy of a Freshness Assertion" title="Direct link to Anatomy of a Freshness Assertion"></a></h3><p>At the most basic level, <strong>Freshness Assertions</strong> consist of a few important parts:</p><ol><li>An <strong>Evaluation Schedule</strong></li><li>A <strong>Change Window</strong></li><li>A <strong>Change Source</strong></li></ol><p>In this section, we'll give an overview of each.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="1-evaluation-schedule">1. Evaluation Schedule<a href="#1-evaluation-schedule" class="hash-link" aria-label="Direct link to 1. Evaluation Schedule" title="Direct link to 1. Evaluation Schedule"></a></h4><p>The <strong>Evaluation Schedule</strong>: This defines how often to check a given warehouse Table for new updates. This should usually
|
||
be configured to match the expected change frequency of the Table, although is can also be more frequently.
|
||
If the Table changes daily, it should be daily. If it changes hourly, it should be hourly. You can also specify specific days of the week, hours in the day, or even
|
||
minutes in an hour.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="2-change-window">2. Change Window<a href="#2-change-window" class="hash-link" aria-label="Direct link to 2. Change Window" title="Direct link to 2. Change Window"></a></h4><p>The <strong>Change Window</strong>: This defines the window of time that is used when determining whether a change has been made to a Table.
|
||
We can either check for change to the Table</p><ul><li><p><em>Since the freshness check was last evaluated</em>. For example, if the evaluation schedule is set to run every day at
|
||
8am PST, we can check whether a change was made between the previous day at 8am and the following day at 8am.</p></li><li><p><em>Within a specific amount of time of the freshness check being evaluated</em> (A fixed interval). For example, if the evaluation schedule is set to run
|
||
every day at 8am PST, we can check whether a change was made in the <em>8 hours before</em> the check is evaluated, which would mean
|
||
in the time between midnight (12:00am) and 8:00am PST.</p></li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="3-change-source">3. Change Source<a href="#3-change-source" class="hash-link" aria-label="Direct link to 3. Change Source" title="Direct link to 3. Change Source"></a></h4><p>The <strong>Change Source</strong>: This is the mechanism that DataHub Cloud should use to determine whether the Table has changed. The supported
|
||
Change Source types vary by the platform, but generally fall into these categories:</p><ul><li><p><strong>Audit Log</strong> (Default): A metadata API or Table that is exposed by the Data Warehouse which contains captures information about the
|
||
operations that have been performed to each Table. It is usually efficient to check, but some useful operations are not
|
||
fully supported across all major Warehouse platforms. Note that for Databricks, <a href="https://docs.databricks.com/en/delta/history.html" target="_blank" rel="noopener noreferrer">this option</a>
|
||
is only available for tables stored in Delta format.</p></li><li><p><strong>Information Schema</strong>: A system Table that is exposed by the Data Warehouse which contains live information about the Databases
|
||
and Tables stored inside the Data Warehouse. It is usually efficient to check, but lacks detailed information about the <em>type</em>
|
||
of change that was last made to a specific table (e.g. the operation itself - INSERT, UPDATE, DELETE, number of impacted rows, etc).
|
||
Note that for Databricks, <a href="https://docs.databricks.com/en/delta/table-details.html" target="_blank" rel="noopener noreferrer">this option</a> is only available for tables stored in Delta format.</p></li><li><p><strong>Last Modified Column</strong>: A Date or Timestamp column that represents the last time that a specific <em>row</em> was touched or updated.
|
||
Adding a Last Modified Column to each warehouse Table is a pattern is often used for existing use cases around change management.
|
||
If this change source is used, a query will be issued to the Table to search for rows that have been modified within a specific
|
||
window of time (based on the Change Window)</p></li><li><p><strong>High Watermark Column</strong>: A column that contains a constantly-incrementing value - a date, a time, or another always-increasing number.
|
||
If this change source is used, a query will be issued to the Table to look for rows with a new "high watermark", e.g. a value that
|
||
is higher than the previously observed value, in order to determine whether the Table has been changed within a given period of time.
|
||
Note that this approach is only supported if the Change Window does not use a fixed interval.</p></li><li><p><strong>DataHub Operation</strong>: A DataHub "Operation" aspect contains timeseries information used to describe changes made to an entity. Using this
|
||
option avoids contacting your data platform, and instead uses the DataHub Operation metadata to evaluate Freshness Assertions.
|
||
This relies on Operations being reported to DataHub, either via ingestion or via use of the DataHub APIs (see <a href="#reporting-operations-via-api">Report Operation via API</a>).
|
||
Note if you have not configured an ingestion source through DataHub, then this may be the only option available. By default, any operation type found will be considered a valid change. Use the <strong>Operation Types</strong> dropdown when selecting this option to specify which operation types should be considered valid changes. You may choose from one of DataHub's standard Operation Types, or specify a "Custom" Operation Type by typing in the name of the Operation Type.</p></li><li><p><strong>File Metadata</strong> (Databricks Only): A column that is exposed by Databricks for both Unity Catalog and Hive Metastore based tables
|
||
which includes information about the last time that a file for the table was changed. Read more about it <a href="https://docs.databricks.com/en/ingestion/file-metadata-column.html" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Using either of the column value approaches (<strong>Last Modified Column</strong> or <strong>High Watermark Column</strong>) to determine whether a Table has changed can be useful because it can be customized to determine whether specific types of changes have been made to a given Table.
|
||
And because this type of assertion does not involve system warehouse tables, they are easily portable across Data Warehouse and Data Lake providers.</p></li></ul><p>Freshness Assertions also have an off switch: they can be started or stopped at any time with the click of button.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="creating-a-freshness-assertion">Creating a Freshness Assertion<a href="#creating-a-freshness-assertion" class="hash-link" aria-label="Direct link to Creating a Freshness Assertion" title="Direct link to Creating a Freshness Assertion"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites"></a></h3><ol><li><strong>Permissions</strong>: To create or delete Freshness Assertions for a specific entity on DataHub, you'll need to be granted the
|
||
<code>Edit Assertions</code> and <code>Edit Monitors</code> privileges for the entity. This will be granted to Entity owners as part of the <code>Asset Owners - Metadata Policy</code>
|
||
by default.</li><li>(Optional) <strong>Data Platform Connection</strong>: In order to create a Freshness Assertion that queries the source data platform directly (instead of DataHub metadata), you'll need to have an <strong>Ingestion Source</strong> configured to your
|
||
Data Platform: Snowflake, BigQuery, or Redshift under the <strong>Integrations</strong> tab.</li></ol><p>Once these are in place, you're ready to create your Freshness Assertions!</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="steps">Steps<a href="#steps" class="hash-link" aria-label="Direct link to Steps" title="Direct link to Steps"></a></h3><ol><li>Navigate to the Table that to monitor for freshness</li><li>Click the <strong>Quality</strong> tab</li></ol><p align="left"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/profile-validation-tab.png" class="img_ev3q"></p><ol start="3"><li>Click <strong>+ Create Assertion</strong></li></ol><p align="left"><img loading="lazy" width="40%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/assertion-builder-choose-type.png" class="img_ev3q"></p><ol start="4"><li><p>Choose <strong>Freshness</strong></p></li><li><p>Configure the evaluation <strong>schedule</strong>. This is the frequency that the table will be checked for changes. This represents you
|
||
expectation about the frequency at which the table should be updated.</p></li><li><p>Configure the evaluation <strong>period</strong>. This defines the period of time that will be considered when looking for changes to the table. Choose between <em>Since the previous check</em> to check whether the table has changed since the past evaluation,
|
||
or <em>In the past X hours</em> to configure a fixed interval that is used when checking the table.</p></li></ol><p><em>Check whether the table has changed between subsequent evaluations of the check</em></p><p align="left"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/assertion-builder-freshness-since-last.png" class="img_ev3q"></p><p><em>Check whether the table has changed in a specific window of time</em></p><p align="left"><img loading="lazy" width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/assertion-builder-freshness-fixed-interval.png" class="img_ev3q"></p><ol start="7"><li>(Optional) Click <strong>Advanced</strong> to customize the evaluation <strong>source</strong>. This is the mechanism that will be used to evaluate
|
||
the check. Each Data Platform supports different options including Audit Log, Information Schema, Last Modified Column, High Watermark Column, and DataHub Operation.</li></ol><p align="left"><img loading="lazy" width="40%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/assertion-builder-freshness-source-type.png" class="img_ev3q"></p><ul><li><strong>Audit Log</strong>: Check the Data Platform operational audit log to determine whether the table changed within the evaluation period.</li><li><strong>Information Schema</strong>: Check the Data Platform system metadata tables to determine whether the table changed within the evaluation period.</li><li><strong>Last Modified Column</strong>: Check for the presence of rows using a "Last Modified Time" column, which should reflect the time at which a given row was last changed in the table, to
|
||
determine whether the table changed within the evaluation period.</li><li><strong>High Watermark Column</strong>: Monitor changes to a continuously-increasing "high watermark" column value to determine whether a table
|
||
has been changed. This option is particularly useful for tables that grow consistently with time, for example fact or event (e.g. click-stream) tables. It is not available
|
||
when using a fixed lookback period.</li><li><strong>DataHub Operation</strong>: Use DataHub Operations to determine whether the table changed within the evaluation period.</li></ul><ol start="8"><li>Configure actions that should be taken when the Freshness Assertion passes or fails</li></ol><p align="left"><img loading="lazy" width="45%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/shared/assertion-builder-actions.png" class="img_ev3q"></p><ul><li><p><strong>Raise incident</strong>: Automatically raise a new DataHub <code>Freshness</code> Incident for the Table whenever the Freshness Assertion is failing. This
|
||
may indicate that the Table is unfit for consumption. Configure Slack Notifications under <strong>Settings</strong> to be notified when
|
||
an incident is created due to an Assertion failure.</p></li><li><p><strong>Resolve incident</strong>: Automatically resolved any incidents that were raised due to failures in this Freshness Assertion. Note that
|
||
any other incidents will not be impacted.</p></li></ul><ol start="9"><li><p>Click <strong>Next</strong> and add a description.</p></li><li><p>Click <strong>Save</strong>.</p></li></ol><p>And that's it! DataHub will now begin to monitor your Freshness Assertion for the table.</p><p>Once your assertion has run, you will begin to see Success or Failure status for the Table</p><p align="left"><img loading="lazy" width="45%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/failing-assertions-section.png" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stopping-a-freshness-assertion">Stopping a Freshness Assertion<a href="#stopping-a-freshness-assertion" class="hash-link" aria-label="Direct link to Stopping a Freshness Assertion" title="Direct link to Stopping a Freshness Assertion"></a></h2><p>In order to temporarily stop the evaluation of the assertion:</p><ol><li>Navigate to the <strong>Quality</strong> tab of the Table with the assertion</li><li>Click <strong>Freshness</strong> to open the Freshness Assertion assertions</li><li>Click the "Stop" button for the assertion you wish to pause.</li></ol><p align="left"><img loading="lazy" width="25%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/shared/stop-assertion.png" class="img_ev3q"></p><p>To resume the assertion, simply click <strong>Start</strong>.</p><p align="left"><img loading="lazy" width="25%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/shared/start-assertion.png" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="anomaly-detection-with-smart-assertions-">Anomaly Detection with Smart Assertions ⚡<a href="#anomaly-detection-with-smart-assertions-" class="hash-link" aria-label="Direct link to Anomaly Detection with Smart Assertions ⚡" title="Direct link to Anomaly Detection with Smart Assertions ⚡"></a></h2><p>As part of the <strong>DataHub Cloud Observe</strong> module, DataHub Cloud also provides <strong>Smart Assertions</strong> out of the box. These are
|
||
dynamic, AI-powered Freshness Assertions that you can use to monitor the freshness of important warehouse Tables, without
|
||
requiring any manual setup. The Smart Assertion's ML model will train based on the Table change history that is captured in the <a href="/docs/api/tutorials/operations"><code>operation</code> aspect</a>. Normally this is populated during ingestion run time.</p><p>You can create smart assertions by simply selecting the <code>Detect with AI</code> option in the UI:</p><p align="left"><img loading="lazy" width="90%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/freshness-smart-assertion.png" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="creating-freshness-assertions-via-api">Creating Freshness Assertions via API<a href="#creating-freshness-assertions-via-api" class="hash-link" aria-label="Direct link to Creating Freshness Assertions via API" title="Direct link to Creating Freshness Assertions via API"></a></h2><p>Under the hood, DataHub Cloud implements Freshness Assertion Monitoring using two concepts:</p><ul><li><strong>Assertion</strong>: The specific expectation for freshness, e.g. "The table was changed int the past 7 hours"
|
||
or "The table is changed on a schedule of every day by 8am". This is the "what".</li><li><strong>Monitor</strong>: The process responsible for evaluating the Assertion on a given evaluation schedule and using specific
|
||
mechanisms. This is the "how".</li></ul><p>Note that to create or delete Assertions and Monitors for a specific entity on DataHub, you'll need the
|
||
<code>Edit Assertions</code> and <code>Edit Monitors</code> privileges for it.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="graphql">GraphQL<a href="#graphql" class="hash-link" aria-label="Direct link to GraphQL" title="Direct link to GraphQL"></a></h4><p>In order to create or update a Freshness Assertion, you can use the <code>upsertDatasetFreshnessAssertionMonitor</code> mutation.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples"></a></h5><p>To create a Freshness Assertion Entity that checks whether a table has been updated in the past 8, and runs every 8 hours:</p><div class="language-graphql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-graphql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">mutation</span><span class="token plain"> </span><span class="token definition-mutation function" style="color:rgb(130, 170, 255)">upsertDatasetFreshnessAssertionMonitor</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property-query">upsertDatasetFreshnessAssertionMonitor</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">input</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">entityUrn</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token description string language-markdown tag punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)">urn</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">of</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">entity</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">being</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">monitored</span><span class="token description string language-markdown tag punctuation" style="color:rgb(199, 146, 234)">></span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">schedule</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">type</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token constant" style="color:rgb(130, 170, 255)">FIXED_INTERVAL</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">fixedInterval</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">unit</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token constant" style="color:rgb(130, 170, 255)">HOUR</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">multiple</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">evaluationSchedule</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">timezone</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token description string language-markdown" style="color:rgb(195, 232, 141)">America/Los_Angeles</span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">cron</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"0 */8 * * *"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">evaluationParameters</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">sourceType</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token constant" style="color:rgb(130, 170, 255)">INFORMATION_SCHEMA</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">mode</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token constant" style="color:rgb(130, 170, 255)">ACTIVE</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">urn</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>To create an AI Smart Freshness Assertion:</p><div class="language-graphql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-graphql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">mutation</span><span class="token plain"> </span><span class="token definition-mutation function" style="color:rgb(130, 170, 255)">upsertDatasetFreshnessAssertionMonitor</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property-query">upsertDatasetFreshnessAssertionMonitor</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">input</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">entityUrn</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token description string language-markdown tag punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)">urn</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">of</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">entity</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">being</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">monitored</span><span class="token description string language-markdown tag punctuation" style="color:rgb(199, 146, 234)">></span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">inferWithAI</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token boolean" style="color:rgb(255, 88, 116)">true</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">evaluationSchedule</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">timezone</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"America/Los_Angeles"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">cron</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"0 * * * *"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">evaluationParameters</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">sourceType</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token constant" style="color:rgb(130, 170, 255)">INFORMATION_SCHEMA</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">mode</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token constant" style="color:rgb(130, 170, 255)">ACTIVE</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">urn</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can use same endpoint with assertion urn input to update an existing Freshness Assertion and corresponding Monitor:</p><div class="language-graphql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-graphql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">mutation</span><span class="token plain"> </span><span class="token definition-mutation function" style="color:rgb(130, 170, 255)">upsertDatasetFreshnessAssertionMonitor</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property-query">upsertDatasetFreshnessAssertionMonitor</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">assertionUrn</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token description string language-markdown tag punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)">urn</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">of</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">assertion</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">created</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">in</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">earlier</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">query</span><span class="token description string language-markdown tag punctuation" style="color:rgb(199, 146, 234)">></span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">input</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">entityUrn</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token description string language-markdown tag punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)">urn</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">of</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">entity</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">being</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">monitored</span><span class="token description string language-markdown tag punctuation" style="color:rgb(199, 146, 234)">></span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">schedule</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">type</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token constant" style="color:rgb(130, 170, 255)">FIXED_INTERVAL</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">fixedInterval</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">unit</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token constant" style="color:rgb(130, 170, 255)">HOUR</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">multiple</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">6</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">evaluationSchedule</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">timezone</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token description string language-markdown" style="color:rgb(195, 232, 141)">America/Los_Angeles</span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">cron</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"0 */6 * * *"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">evaluationParameters</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">sourceType</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token constant" style="color:rgb(130, 170, 255)">INFORMATION_SCHEMA</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">mode</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token constant" style="color:rgb(130, 170, 255)">ACTIVE</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">urn</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can delete assertions along with their monitors using GraphQL mutations: <code>deleteAssertion</code> and <code>deleteMonitor</code>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reporting-operations-via-api">Reporting Operations via API<a href="#reporting-operations-via-api" class="hash-link" aria-label="Direct link to Reporting Operations via API" title="Direct link to Reporting Operations via API"></a></h3><p>DataHub Operations can be used to capture changes made to entities. This is useful for cases where the underlying data platform does not provide a mechanism
|
||
to capture changes, or where the data platform's mechanism is not reliable. In order to report an operation, you can use the <code>reportOperation</code> GraphQL mutation.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="examples-1">Examples<a href="#examples-1" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples"></a></h5><div class="language-graphql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-graphql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">mutation</span><span class="token plain"> </span><span class="token definition-mutation function" style="color:rgb(130, 170, 255)">reportOperation</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property-query">reportOperation</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">input</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">urn</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token description string language-markdown tag punctuation" style="color:rgb(199, 146, 234)"><</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)">urn</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">of</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">the</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">dataset</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">being</span><span class="token description string language-markdown tag" style="color:rgb(255, 85, 114)"> </span><span class="token description string language-markdown tag attr-name" style="color:rgb(255, 203, 107)">reported</span><span class="token description string language-markdown tag punctuation" style="color:rgb(199, 146, 234)">></span><span class="token description string" style="color:rgb(195, 232, 141)">"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">operationType</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token constant" style="color:rgb(130, 170, 255)">INSERT</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">sourceType</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token constant" style="color:rgb(130, 170, 255)">DATA_PLATFORM</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token attr-name" style="color:rgb(255, 203, 107)">timestampMillis</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1693252366489</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the <code>timestampMillis</code> field to specify the time at which the operation occurred. If no value is provided, the current time will be used.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tips">Tips<a href="#tips" class="hash-link" aria-label="Direct link to Tips" title="Direct link to Tips"></a></h3><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p><strong>Authorization</strong></p><p>Remember to always provide a DataHub Personal Access Token when calling the GraphQL API. To do so, just add the 'Authorization' header as follows:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">Authorization: Bearer <personal-access-token></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><strong>Exploring GraphQL API</strong></p><p>Also, remember that you can play with an interactive version of the DataHub Cloud GraphQL API at <code>https://your-account-id.acryl.io/api/graphiql</code></p></div></div></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="slackUtm_uoBr"><div class="slackUtm_uoBr"><hr>Need more help? Join the conversation in <a href="https://datahub.com/slack?utm_source=docs&utm_medium=footer&utm_campaign=docs_footer&utm_content=docs/managed-datahub/observe/freshness-assertions">Slack!</a></div></div><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/freshness-assertions.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_VsjB"></div></div></footer><div class="feedbackWrapper_mUHF"><div class="feedbackWidget_PX4d"><div class="feedbackButtons_wn3V"><strong>Is this page helpful?</strong><div><button class="feedbackButton_UgQs"><span role="img" aria-label="like" class="anticon anticon-like"><svg viewBox="64 64 896 896" focusable="false" data-icon="like" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M885.9 533.7c16.8-22.2 26.1-49.4 26.1-77.7 0-44.9-25.1-87.4-65.5-111.1a67.67 67.67 0 00-34.3-9.3H572.4l6-122.9c1.4-29.7-9.1-57.9-29.5-79.4A106.62 106.62 0 00471 99.9c-52 0-98 35-111.8 85.1l-85.9 311H144c-17.7 0-32 14.3-32 32v364c0 17.7 14.3 32 32 32h601.3c9.2 0 18.2-1.8 26.5-5.4 47.6-20.3 78.3-66.8 78.3-118.4 0-12.6-1.8-25-5.4-37 16.8-22.2 26.1-49.4 26.1-77.7 0-12.6-1.8-25-5.4-37 16.8-22.2 26.1-49.4 26.1-77.7-.2-12.6-2-25.1-5.6-37.1zM184 852V568h81v284h-81zm636.4-353l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 16.5-7.2 32.2-19.6 43l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 16.5-7.2 32.2-19.6 43l-21.9 19 13.9 25.4a56.2 56.2 0 016.9 27.3c0 22.4-13.2 42.6-33.6 51.8H329V564.8l99.5-360.5a44.1 44.1 0 0142.2-32.3c7.6 0 15.1 2.2 21.1 6.7 9.9 7.4 15.2 18.6 14.6 30.5l-9.6 198.4h314.4C829 418.5 840 436.9 840 456c0 16.5-7.2 32.1-19.6 43z"></path></svg></span></button><button class="feedbackButton_UgQs"><span role="img" aria-label="dislike" class="anticon anticon-dislike"><svg viewBox="64 64 896 896" focusable="false" data-icon="dislike" width="1em" height="1em" fill="currentColor" aria-hidden="true"><path d="M885.9 490.3c3.6-12 5.4-24.4 5.4-37 0-28.3-9.3-55.5-26.1-77.7 3.6-12 5.4-24.4 5.4-37 0-28.3-9.3-55.5-26.1-77.7 3.6-12 5.4-24.4 5.4-37 0-51.6-30.7-98.1-78.3-118.4a66.1 66.1 0 00-26.5-5.4H144c-17.7 0-32 14.3-32 32v364c0 17.7 14.3 32 32 32h129.3l85.8 310.8C372.9 889 418.9 924 470.9 924c29.7 0 57.4-11.8 77.9-33.4 20.5-21.5 31-49.7 29.5-79.4l-6-122.9h239.9c12.1 0 23.9-3.2 34.3-9.3 40.4-23.5 65.5-66.1 65.5-111 0-28.3-9.3-55.5-26.1-77.7zM184 456V172h81v284h-81zm627.2 160.4H496.8l9.6 198.4c.6 11.9-4.7 23.1-14.6 30.5-6.1 4.5-13.6 6.8-21.1 6.7a44.28 44.28 0 01-42.2-32.3L329 459.2V172h415.4a56.85 56.85 0 0133.6 51.8c0 9.7-2.3 18.9-6.9 27.3l-13.9 25.4 21.9 19a56.76 56.76 0 0119.6 43c0 9.7-2.3 18.9-6.9 27.3l-13.9 25.4 21.9 19a56.76 56.76 0 0119.6 43c0 9.7-2.3 18.9-6.9 27.3l-14 25.5 21.9 19a56.76 56.76 0 0119.6 43c0 19.1-11 37.5-28.8 48.4z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/managed-datahub/observe/custom-sql-assertions"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Custom SQL Assertions</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/managed-datahub/observe/schema-assertions"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Schema Assertions</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#introduction" class="table-of-contents__link toc-highlight">Introduction</a></li><li><a href="#support" class="table-of-contents__link toc-highlight">Support</a></li><li><a href="#what-is-a-freshness-assertion" class="table-of-contents__link toc-highlight">What is a Freshness Assertion?</a><ul><li><a href="#anatomy-of-a-freshness-assertion" class="table-of-contents__link toc-highlight">Anatomy of a Freshness Assertion</a></li></ul></li><li><a href="#creating-a-freshness-assertion" class="table-of-contents__link toc-highlight">Creating a Freshness Assertion</a><ul><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#steps" class="table-of-contents__link toc-highlight">Steps</a></li></ul></li><li><a href="#stopping-a-freshness-assertion" class="table-of-contents__link toc-highlight">Stopping a Freshness Assertion</a></li><li><a href="#anomaly-detection-with-smart-assertions-" class="table-of-contents__link toc-highlight">Anomaly Detection with Smart Assertions ⚡</a></li><li><a href="#creating-freshness-assertions-via-api" class="table-of-contents__link toc-highlight">Creating Freshness Assertions via API</a><ul><li><a href="#reporting-operations-via-api" class="table-of-contents__link toc-highlight">Reporting Operations via API</a></li><li><a href="#tips" class="table-of-contents__link toc-highlight">Tips</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/docs/">Introduction</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/quickstart">Quickstart</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://datahub.com/slack" target="_blank" rel="noopener noreferrer" class="footer__link-item">Slack<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w" target="_blank" rel="noopener noreferrer" class="footer__link-item">YouTube<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://medium.com/datahub-project" target="_blank" rel="noopener noreferrer" class="footer__link-item">Blog<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/docs/townhalls">Town Halls</a></li><li class="footer__item"><a href="https://datahub.com/adoption-stories/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Adoption<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://demo.datahub.com/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Demo</a></li><li class="footer__item"><a href="https://feature-requests.datahubproject.io/roadmap" target="_blank" rel="noopener noreferrer" class="footer__link-item">Roadmap<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/docs/contributing">Contributing</a></li><li class="footer__item"><a href="https://github.com/datahub-project/datahub" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://feature-requests.datahubproject.io/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Feature Requests<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2015-2025 DataHub Project Authors.</div></div></div></footer></div>
|
||
<script src="/assets/js/runtime~main.8ae4198a.js"></script>
|
||
<script src="/assets/js/main.9d79f7e2.js"></script>
|
||
</body>
|
||
</html> |