datahub/assets/js/0b513daf.0de01fcd.js

"use strict";(self.webpackChunkdocs_website=self.webpackChunkdocs_website||[]).push([[2439],{15680:(e,t,r)=>{r.d(t,{xA:()=>b,yg:()=>d});var a=r(96540);function n(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,a)}return r}function s(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{};t%2?o(Object(r),!0).forEach((function(t){n(e,t,r[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):o(Object(r)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(r,t))}))}return e}function c(e,t){if(null==e)return{};var r,a,n=function(e,t){if(null==e)return{};var r,a,n={},o=Object.keys(e);for(a=0;a<o.length;a++)r=o[a],t.indexOf(r)>=0||(n[r]=e[r]);return n}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)r=o[a],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(n[r]=e[r])}return n}var i=a.createContext({}),u=function(e){var t=a.useContext(i),r=t;return e&&(r="function"==typeof e?e(t):s(s({},t),e)),r},b=function(e){var t=u(e.components);return a.createElement(i.Provider,{value:t},e.children)},l="mdxType",p={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},m=a.forwardRef((function(e,t){var r=e.components,n=e.mdxType,o=e.originalType,i=e.parentName,b=c(e,["components","mdxType","originalType","parentName"]),l=u(r),m=n,d=l["".concat(i,".").concat(m)]||l[m]||p[m]||o;return r?a.createElement(d,s(s({ref:t},b),{},{components:r})):a.createElement(d,s({ref:t},b))}));function d(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=r.length,s=new Array(o);s[0]=m;var c={};for(var i in t)hasOwnProperty.call(t,i)&&(c[i]=t[i]);c.originalType=e,c[l]="string"==typeof e?e:n,s[1]=c;for(var u=2;u<o;u++)s[u]=r[u];return a.createElement.apply(null,s)}return a.createElement.apply(null,r)}m.displayName="MDXCreateElement"},50917:(e,t,r)=>{r.r(t),r.d(t,{assets:()=>b,contentTitle:()=>i,default:()=>d,frontMatter:()=>c,metadata:()=>u,toc:()=>l});r(96540);var a=r(15680);function n(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){return t=null!=t?t:{},Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):function(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,a)}return r}(Object(t)).forEach((function(r){Object.defineProperty(e,r,Object.getOwnPropertyDescriptor(t,r))})),e}function s(e,t){if(null==e)return{};var r,a,n=function(e,t){if(null==e)return{};var r,a,n={},o=Object.keys(e);for(a=0;a<o.length;a++)r=o[a],t.indexOf(r)>=0||(n[r]=e[r]);return n}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)r=o[a],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(n[r]=e[r])}return n}const c={title:"MXE Processing Jobs",slug:"/metadata-jobs",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/metadata-jobs/README.md"},i="MXE Processing Jobs",u={unversionedId:"metadata-jobs/README",id:"metadata-jobs/README",title:"MXE Processing Jobs",description:"DataHub uses Kafka as the pub-sub message queue in the backend. There are 2 Kafka topics used by DataHub which are",source:"@site/genDocs/metadata-jobs/README.md",sourceDirName:"metadata-jobs",slug:"/metadata-jobs",permalink:"/docs/metadata-jobs",draft:!1,editUrl:"https://github.com/datahub-project/datahub/blob/master/metadata-jobs/README.md",tags:[],version:"current",frontMatter:{title:"MXE Processing Jobs",slug:"/metadata-jobs",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/metadata-jobs/README.md"}},b={},l=[],p={toc:l},m="wrapper";function d(e){var{components:t}=e,r=s(e,["components"]);return(0,a.yg)(m,o(function(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{},a=Object.keys(r);"function"==typeof Object.getOwnPropertySymbols&&(a=a.concat(Object.getOwnPropertySymbols(r).filter((function(e){return Object.getOwnPropertyDescriptor(r,e).enumerable})))),a.forEach((function(t){n(e,t,r[t])}))}return e}({},p,r),{components:t,mdxType:"MDXLayout"}),(0,a.yg)("h1",{id:"mxe-processing-jobs"},"MXE Processing Jobs"),(0,a.yg)("p",null,"DataHub uses Kafka as the pub-sub message queue in the backend. There are 2 Kafka topics used by DataHub which are\n",(0,a.yg)("inlineCode",{parentName:"p"},"MetadataChangeEvent")," and ",(0,a.yg)("inlineCode",{parentName:"p"},"MetadataAuditEvent"),"."),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("inlineCode",{parentName:"li"},"MetadataChangeEvent:")," This message is emitted by any data platform or crawler in which there is a change in the metadata."),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("inlineCode",{parentName:"li"},"MetadataAuditEvent:")," This message is emitted by ",(0,a.yg)("a",{parentName:"li",href:"/docs/metadata-service"},"DataHub GMS")," to notify that metadata change is registered.")),(0,a.yg)("p",null,"To be able to consume from these two topics, there are two Spring\njobs DataHub uses:"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("a",{parentName:"li",href:"https://github.com/datahub-project/datahub/blob/master/metadata-jobs/mce-consumer-job"},"MCE Consumer Job"),": Writes to ",(0,a.yg)("a",{parentName:"li",href:"/docs/metadata-service"},"DataHub GMS")),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("a",{parentName:"li",href:"https://github.com/datahub-project/datahub/blob/master/metadata-jobs/mae-consumer-job"},"MAE Consumer Job"),": Writes to ",(0,a.yg)("a",{parentName:"li",href:"https://github.com/datahub-project/datahub/blob/master/docker/elasticsearch"},"Elasticsearch")," & ",(0,a.yg)("a",{parentName:"li",href:"https://github.com/datahub-project/datahub/blob/master/docker/neo4j"},"Neo4j"))))}d.isMDXComponent=!0}}]);