"use strict";(self.webpackChunkdocs_website=self.webpackChunkdocs_website||[]).push([[89037],{15680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>m});var r=a(96540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function n(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function s(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?n(Object(a),!0).forEach((function(t){o(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):n(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function i(e,t){if(null==e)return{};var a,r,o=function(e,t){if(null==e)return{};var a,r,o={},n=Object.keys(e);for(r=0;r<n.length;r++)a=n[r],t.indexOf(a)>=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(r=0;r<n.length;r++)a=n[r],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var l=r.createContext({}),u=function(e){var t=r.useContext(l),a=t;return e&&(a="function"==typeof e?e(t):s(s({},t),e)),a},c=function(e){var t=u(e.components);return r.createElement(l.Provider,{value:t},e.children)},d="mdxType",h={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},p=r.forwardRef((function(e,t){var a=e.components,o=e.mdxType,n=e.originalType,l=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),d=u(a),p=o,m=d["".concat(l,".").concat(p)]||d[p]||h[p]||n;return a?r.createElement(m,s(s({ref:t},c),{},{components:a})):r.createElement(m,s({ref:t},c))}));function m(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var n=a.length,s=new Array(n);s[0]=p;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i[d]="string"==typeof e?e:o,s[1]=i;for(var u=2;u<n;u++)s[u]=a[u];return r.createElement.apply(null,s)}return r.createElement.apply(null,a)}p.displayName="MDXCreateElement"},95536:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>l,default:()=>m,frontMatter:()=>i,metadata:()=>u,toc:()=>d});a(96540);var r=a(15680);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function n(e,t){return t=null!=t?t:{},Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):function(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}(Object(t)).forEach((function(a){Object.defineProperty(e,a,Object.getOwnPropertyDescriptor(t,a))})),e}function s(e,t){if(null==e)return{};var a,r,o=function(e,t){if(null==e)return{};var a,r,o={},n=Object.keys(e);for(r=0;r<n.length;r++)a=n[r],t.indexOf(a)>=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(r=0;r<n.length;r++)a=n[r],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}const i={title:"Assertions",slug:"/managed-datahub/observe/assertions",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/assertions.md"},l="Assertions",u={unversionedId:"docs/managed-datahub/observe/assertions",id:"docs/managed-datahub/observe/assertions",title:"Assertions",description:"Currently we support monitoring data on Snowflake, Redshift, BigQuery, and Databricks as part of DataHub Cloud Observe.",source:"@site/genDocs/docs/managed-datahub/observe/assertions.md",sourceDirName:"docs/managed-datahub/observe",slug:"/managed-datahub/observe/assertions",permalink:"/docs/managed-datahub/observe/assertions",draft:!1,editUrl:"https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/assertions.md",tags:[],version:"current",frontMatter:{title:"Assertions",slug:"/managed-datahub/observe/assertions",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/assertions.md"},sidebar:"overviewSidebar",previous:{title:"Features",permalink:"/docs/category/features"},next:{title:"Assertions",permalink:"/docs/managed-datahub/observe/assertions"}},c={},d=[{value:"How to Create and Run Assertions",id:"how-to-create-and-run-assertions",level:2},{value:"DataHub Cloud Assertions",id:"datahub-cloud-assertions",level:3},{value:"Bulk Creating Assertions",id:"bulk-creating-assertions",level:4},{value:"AI Anomaly Detection (Smart Assertions)",id:"ai-anomaly-detection-smart-assertions",level:3},{value:"The Smart Assertion Solution",id:"the-smart-assertion-solution",level:3},{value:"Reporting from 3rd Party tools",id:"reporting-from-3rd-party-tools",level:3},{value:"Alerts",id:"alerts",level:2},{value:"Sifting through the noise &amp; Data Health Reporting",id:"sifting-through-the-noise--data-health-reporting",level:2},{value:"Cost",id:"cost",level:2},{value:"Execution details - Where and How",id:"execution-details---where-and-how",level:2},{value:"Privacy: Execute In-Network, avoid exposing data externally",id:"privacy-execute-in-network-avoid-exposing-data-externally",level:3},{value:"Source system selection",id:"source-system-selection",level:3}],h={toc:d},p="wrapper";function m(e){var{components:t}=e,a=s(e,["components"]);return(0,r.yg)(p,n(function(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{},r=Object.keys(a);"function"==typeof Object.getOwnPropertySymbols&&(r=r.concat(Object.getOwnPropertySymbols(a).filter((function(e){return Object.getOwnPropertyDescriptor(a,e).enumerable})))),r.forEach((function(t){o(e,t,a[t])}))}return e}({},h,a),{components:t,mdxType:"MDXLayout"}),(0,r.yg)("h1",{id:"assertions"},"Assertions"),(0,r.yg)("admonition",{title:"Supported Data Platforms",type:"note"},(0,r.yg)("p",{parentName:"admonition"},"Currently we support monitoring data on Snowflake, Redshift, BigQuery, and Databricks as part of DataHub Cloud Observe.\nFor other data platforms, DataHub Cloud Observe can monitor assertions against dataset metrics (such as volume, or column nullness) and dataset freshenss by using the ingested statistics for each asset.\nColumn Value and Custom SQL Assertions are not currently supported for other data platforms.")),(0,r.yg)("p",null,"An assertion is ",(0,r.yg)("strong",{parentName:"p"},"a data quality test that finds data that violates a specified rule."),"\nAssertions serve as the building blocks of ",(0,r.yg)("a",{parentName:"p",href:"/docs/managed-datahub/observe/data-contract"},"Data Contracts")," \u2013 this is how we verify the contract is met."),(0,r.yg)("h2",{id:"how-to-create-and-run-assertions"},"How to Create and Run Assertions"),(0,r.yg)("p",null,"Data quality tests (a.k.a. assertions) can be created and run by DataHub Cloud or ingested from a 3rd party tool."),(0,r.yg)("h3",{id:"datahub-cloud-assertions"},"DataHub Cloud Assertions"),(0,r.yg)("p",null,"For DataHub-provided assertion runners, we can deploy an agent in your environment to hit your sources and DataHub. DataHub Cloud Observe offers out-of-the-box evaluation of the following kinds of assertions:"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"/docs/managed-datahub/observe/freshness-assertions"},"Freshness")," (SLAs)"),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"/docs/managed-datahub/observe/volume-assertions"},"Volume")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"/docs/managed-datahub/observe/custom-sql-assertions"},"Custom SQL")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"/docs/managed-datahub/observe/column-assertions"},"Column")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"/docs/managed-datahub/observe/schema-assertions"},"Schema"))),(0,r.yg)("h4",{id:"bulk-creating-assertions"},"Bulk Creating Assertions"),(0,r.yg)("p",null,"You can bulk create Freshness and Volume ",(0,r.yg)("a",{parentName:"p",href:"/docs/managed-datahub/observe/smart-assertions"},"Smart Assertions")," (AI Anomaly Monitors) across several tables at once via the ",(0,r.yg)("a",{parentName:"p",href:"/docs/managed-datahub/observe/data-health-dashboard"},"Data Health Dashboard"),":"),(0,r.yg)("div",{align:"center"},(0,r.yg)("iframe",{width:"560",height:"315",src:"https://www.loom.com/embed/f6720541914645aab6b28cdff8695d9f?sid=58dff84d-bb88-4f02-b814-17fb4986ad1f",frameborder:"0",webkitallowfullscreen:!0,mozallowfullscreen:!0,allowfullscreen:!0})),(0,r.yg)("p",null,"To bulk create column metric assertions on a given dataset, follow the steps under the ",(0,r.yg)("strong",{parentName:"p"},"Anomaly Detection")," section of ",(0,r.yg)("a",{parentName:"p",href:"/docs/managed-datahub/observe/column-assertions#anomaly-detection-with-smart-assertions-"},"Column Assertion"),"."),(0,r.yg)("h3",{id:"ai-anomaly-detection-smart-assertions"},"AI Anomaly Detection (Smart Assertions)"),(0,r.yg)("p",null,"There are many cases where either you do not have the time to figure out what a good rule for an assertion is, or strict rules simply do not suffice for your data validation needs. Traditional rule-based assertions can become inadequate when dealing with complex data patterns or large-scale operations."),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"Common Scenarios")),(0,r.yg)("p",null,"Here are some typical situations where manual assertion rules fall short:"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},(0,r.yg)("strong",{parentName:"p"},"Seasonal data patterns")," - A table whose row count changes exhibit weekly seasonality may need a different set of assertions for each day of the week, making static rules impractical to maintain.")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},(0,r.yg)("strong",{parentName:"p"},"Statistical complexity across large datasets")," - Figuring out what the expected standard deviation is for each column can be incredibly time consuming and not feasible across hundreds of tables, especially when each table has unique characteristics.")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},(0,r.yg)("strong",{parentName:"p"},"Dynamic data environments")," - When data patterns evolve over time, manually updating assertion rules becomes a maintenance burden that can lead to false positives or missed anomalies."))),(0,r.yg)("h3",{id:"the-smart-assertion-solution"},"The Smart Assertion Solution"),(0,r.yg)("p",null,"In these scenarios, you may want to consider creating a ",(0,r.yg)("a",{parentName:"p",href:"/docs/managed-datahub/observe/smart-assertions"},"Smart Assertion")," to let machine learning automatically detect the normal patterns in your data and alert you when anomalies occur. This approach allows for more flexible and adaptive data quality monitoring without the overhead of manual rule maintenance."),(0,r.yg)("p",null,"Both traditional and smart assertions can be defined through the DataHub API or the UI."),(0,r.yg)("p",{align:"center"},(0,r.yg)("img",{width:"70%",src:"https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/assertions/assertion-ui.png"})),(0,r.yg)("h3",{id:"reporting-from-3rd-party-tools"},"Reporting from 3rd Party tools"),(0,r.yg)("p",null,"You can integrate 3rd party tools as follows:"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"/docs/generated/ingestion/sources/dbt#integrating-with-dbt-test"},"DBT Test")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"/docs/metadata-ingestion/integration_docs/great-expectations"},"Great Expectations")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"/docs/api/tutorials/custom-assertions"},"Custom Assertions"))),(0,r.yg)("p",null,"If you opt for a 3rd party tool, it will be your responsibility to ensure the assertions are run based on the Data Contract spec stored in DataHub. With 3rd party runners, you can get the Assertion Change events by subscribing to our Kafka topic using the ",(0,r.yg)("a",{parentName:"p",href:"/docs/actions"},"DataHub Actions Framework"),"."),(0,r.yg)("h2",{id:"alerts"},"Alerts"),(0,r.yg)("p",null,"Beyond the ability to see the results of the assertion checks (and history of the results) both on the physical asset\u2019s page in the DataHub UI and as the result of DataHub API calls, you can also get notified via ",(0,r.yg)("a",{parentName:"p",href:"/docs/managed-datahub/slack/saas-slack-setup"},"Slack messages")," (DMs or to a team channel) based on your ",(0,r.yg)("a",{parentName:"p",href:"https://youtu.be/VNNZpkjHG_I?t=79"},"subscription")," to an assertion run event, or when an ",(0,r.yg)("a",{parentName:"p",href:"/docs/incidents/incidents"},"incident")," is raised or resolved. In the future, we\u2019ll also provide the ability to subscribe directly to contracts."),(0,r.yg)("p",null,"With DataHub Cloud Observe, you can react to the Assertion Run Event by listening to API events via ",(0,r.yg)("a",{parentName:"p",href:"/docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge"},"AWS EventBridge")," (the availability and simplicity of setup of each solution dependent on your current DataHub Cloud setup \u2013 chat with your DataHub Cloud representative to learn more)."),(0,r.yg)("h2",{id:"sifting-through-the-noise--data-health-reporting"},"Sifting through the noise & Data Health Reporting"),(0,r.yg)("p",null,"Sometimes alerts can get noisy, and it's hard to sift through slack notifications to figure out what's important. Sometimes you need to figure out which of the tables your team owns actually have data quality checks running on them.\nThe ",(0,r.yg)("a",{parentName:"p",href:"/docs/managed-datahub/observe/data-health-dashboard"},"Data Health Dashboard")," provides a birds-eye view of the health of your data landscape. You can slice and dice the data to find the exact answers you're looking for."),(0,r.yg)("h2",{id:"cost"},"Cost"),(0,r.yg)("p",null,"We provide a plethora of ways to run your assertions, aiming to allow you to use the cheapest possible means to do so and/or the most accurate means to do so, depending on your use case. For example, for Freshness (SLA) assertions, it is relatively cheap to use either their Audit Log or Information Schema as a means to run freshness checks, and we support both of those as well as Last Modified Column, High Watermark Column, and DataHub Operation (",(0,r.yg)("a",{parentName:"p",href:"/docs/managed-datahub/observe/freshness-assertions#3-change-source"},"see the docs for more details"),")."),(0,r.yg)("h2",{id:"execution-details---where-and-how"},"Execution details - Where and How"),(0,r.yg)("p",null,"There are a few ways DataHub Cloud assertions can be executed:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Directly query the source system:\na. ",(0,r.yg)("inlineCode",{parentName:"li"},"Information Schema")," tables are used by default to power cheap, fast checks on a table's freshness or row count.\nb. ",(0,r.yg)("inlineCode",{parentName:"li"},"Audit log")," or ",(0,r.yg)("inlineCode",{parentName:"li"},"Operation log")," tables can be used to granularly monitor table operations.\nc. The table itself can also be queried directly. This is useful for freshness checks referencing ",(0,r.yg)("inlineCode",{parentName:"li"},"last_updated")," columns, row count checks targetting a subset of the data, and column value checks. We offer several optimizations to reduce query costs for these checks."),(0,r.yg)("li",{parentName:"ol"},"Reference DataHub metadata\na. ",(0,r.yg)("a",{parentName:"li",href:"/docs/api/tutorials/operations"},"Operations")," that are reported via ingestion or our SDKs can power monitoring table freshness.\nb. ",(0,r.yg)("inlineCode",{parentName:"li"},"DatasetProfile")," and ",(0,r.yg)("inlineCode",{parentName:"li"},"SchemaFieldProfile")," ingested or reported via SDKs can power monitoring table metrics and column metrics.")),(0,r.yg)("h3",{id:"privacy-execute-in-network-avoid-exposing-data-externally"},"Privacy: Execute In-Network, avoid exposing data externally"),(0,r.yg)("p",null,"As a part of DataHub Cloud, we offer a ",(0,r.yg)("a",{parentName:"p",href:"/docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor"},"Remote Executor")," deployment model. If this model is used, assertions will execute within your network, and only the results will be sent back to DataHub Cloud. Neither your actual credentials, nor your source data will leave your network."),(0,r.yg)("h3",{id:"source-system-selection"},"Source system selection"),(0,r.yg)("p",null,"Assertions will execute queries using the same source system that was used to initially ingest the table.\nThere are some scenarios where customers may have multiple ingestion sources for, i.e. a BigQuery table. In this case, by default the executor will take the ingestion source that was used to ingest the table's ",(0,r.yg)("inlineCode",{parentName:"p"},"DatasetProperties"),". This behavior can be modified by your customer success rep."))}m.isMDXComponent=!0}}]);