"use strict";(self.webpackChunkdocs_website=self.webpackChunkdocs_website||[]).push([[98176],{15680:(e,t,n)=>{n.d(t,{xA:()=>d,yg:()=>g});var a=n(96540);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var p=a.createContext({}),s=function(e){var t=a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=s(e.components);return a.createElement(p.Provider,{value:t},e.children)},c="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},u=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,p=e.parentName,d=l(e,["components","mdxType","originalType","parentName"]),c=s(n),u=r,g=c["".concat(p,".").concat(u)]||c[u]||m[u]||o;return n?a.createElement(g,i(i({ref:t},d),{},{components:n})):a.createElement(g,i({ref:t},d))}));function g(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,i=new Array(o);i[0]=u;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l[c]="string"==typeof e?e:r,i[1]=l;for(var s=2;s<o;s++)i[s]=n[s];return a.createElement.apply(null,i)}return a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},46113:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>d,contentTitle:()=>p,default:()=>g,frontMatter:()=>l,metadata:()=>s,toc:()=>c});n(96540);var a=n(15680);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){return t=null!=t?t:{},Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):function(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))})),e}function i(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}const l={title:"Bootstrap MetadataChangeProposals (MCPs)",slug:"/advanced/bootstrap-mcps",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/advanced/bootstrap-mcps.md"},p="Bootstrap MetadataChangeProposals (MCPs)",s={unversionedId:"docs/advanced/bootstrap-mcps",id:"docs/advanced/bootstrap-mcps",title:"Bootstrap MetadataChangeProposals (MCPs)",description:"Bootstrap MCPs are templated MCPs which are loaded when the system-update job runs. This allows adding",source:"@site/genDocs/docs/advanced/bootstrap-mcps.md",sourceDirName:"docs/advanced",slug:"/advanced/bootstrap-mcps",permalink:"/docs/advanced/bootstrap-mcps",draft:!1,editUrl:"https://github.com/datahub-project/datahub/blob/master/docs/advanced/bootstrap-mcps.md",tags:[],version:"current",frontMatter:{title:"Bootstrap MetadataChangeProposals (MCPs)",slug:"/advanced/bootstrap-mcps",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/advanced/bootstrap-mcps.md"},sidebar:"overviewSidebar",previous:{title:"Plugins Guide",permalink:"/docs/plugins"},next:{title:"Community",permalink:"/docs/category/community"}},d={},c=[{value:"Process Overview",id:"process-overview",level:2},{value:"<code>bootstrap_mcps.yaml</code> Configuration",id:"bootstrap_mcpsyaml-configuration",level:2},{value:"Template MCPs",id:"template-mcps",level:2},{value:"Example: Native Group",id:"example-native-group",level:3},{value:"Default values",id:"default-values",level:4},{value:"AuditStamp",id:"auditstamp",level:4},{value:"Ingestion Template MCPs",id:"ingestion-template-mcps",level:3},{value:"<code>bootstrap_mcps.yaml</code> Override",id:"bootstrap_mcpsyaml-override",level:2},{value:"Known Limitations",id:"known-limitations",level:2}],m={toc:c},u="wrapper";function g(e){var{components:t}=e,n=i(e,["components"]);return(0,a.yg)(u,o(function(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{},a=Object.keys(n);"function"==typeof Object.getOwnPropertySymbols&&(a=a.concat(Object.getOwnPropertySymbols(n).filter((function(e){return Object.getOwnPropertyDescriptor(n,e).enumerable})))),a.forEach((function(t){r(e,t,n[t])}))}return e}({},m,n),{components:t,mdxType:"MDXLayout"}),(0,a.yg)("h1",{id:"bootstrap-metadatachangeproposals-mcps"},"Bootstrap MetadataChangeProposals (MCPs)"),(0,a.yg)("p",null,"Bootstrap MCPs are templated MCPs which are loaded when the ",(0,a.yg)("inlineCode",{parentName:"p"},"system-update")," job runs. This allows adding\nentities and aspects to DataHub at install time with the ability to customize them via environment variable\noverrides."),(0,a.yg)("p",null,"The built-in bootstrap MCP process can also be extended with custom MCPs. This can streamline deployment\nscenarios where a set of standard ingestion recipes, data platforms, users groups, or other configuration\ncan be applied without the need for developing custom scripts."),(0,a.yg)("h2",{id:"process-overview"},"Process Overview"),(0,a.yg)("p",null,"When DataHub is installed or upgraded, a job runs called ",(0,a.yg)("inlineCode",{parentName:"p"},"system-update"),", this job is responsible for data\nmigration (particularly Elasticsearch indices) and ensuring the data is prepared for the next version of\nDataHub. This is the job which will also apply the bootstrap MCPs."),(0,a.yg)("p",null,"The ",(0,a.yg)("inlineCode",{parentName:"p"},"system-update")," job, depending on configuration, can be split into two sequences of steps. If they are\nnot split, then all steps are blocking."),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"An initial blocking sequence which is run prior to the new version of GMS and other components"),(0,a.yg)("li",{parentName:"ol"},"Second sequence of steps where GMS and other components are allowed to run while additional data migration steps are\ncontinued in the background")),(0,a.yg)("p",null,"When applying bootstrap MCPs ",(0,a.yg)("inlineCode",{parentName:"p"},"system-update")," will perform the following steps:"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"The ",(0,a.yg)("inlineCode",{parentName:"li"},"bootstrap_mcps.yaml")," file is read, either from a default classpath location, ",(0,a.yg)("inlineCode",{parentName:"li"},"bootstrap_mcps.yaml"),", or a filesystem location\nprovided by an environment variable, ",(0,a.yg)("inlineCode",{parentName:"li"},"SYSTEM_UPDATE_BOOTSTRAP_MCP_CONFIG"),"."),(0,a.yg)("li",{parentName:"ol"},"Depending on the mode of blocking or non-blocking each entry in the configuration file will be executed in sequence."),(0,a.yg)("li",{parentName:"ol"},"The template MCP file is loaded either from the classpath, or a filesystem location, and the template values are applied."),(0,a.yg)("li",{parentName:"ol"},"The rendered template MCPs are executed with the options specified in the ",(0,a.yg)("inlineCode",{parentName:"li"},"bootstrap_mcps.yaml"),".")),(0,a.yg)("h2",{id:"bootstrap_mcpsyaml-configuration"},(0,a.yg)("inlineCode",{parentName:"h2"},"bootstrap_mcps.yaml")," Configuration"),(0,a.yg)("p",null,"The ",(0,a.yg)("inlineCode",{parentName:"p"},"bootstrap_mcps.yaml")," file has the following format."),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},"bootstrap:\n  templates:\n    - name: <name>\n      version: <version>\n      force: false\n      blocking: false\n      async: true\n      optional: false\n      mcps_location: <classpath or file location>\n      values_env: <environment variable>\n")),(0,a.yg)("p",null,"Each entry in the list of templates points to a single yaml file which can contain one or more MCP objects. The\nexecution of the template MCPs is tracked by name and version to prevent re-execution. The MCP objects are executed once\nunless ",(0,a.yg)("inlineCode",{parentName:"p"},"force=true")," for each ",(0,a.yg)("inlineCode",{parentName:"p"},"name"),"/",(0,a.yg)("inlineCode",{parentName:"p"},"version")," combination."),(0,a.yg)("p",null,"See the following table of options for descriptions of each field in the template configuration."),(0,a.yg)("table",null,(0,a.yg)("thead",{parentName:"table"},(0,a.yg)("tr",{parentName:"thead"},(0,a.yg)("th",{parentName:"tr",align:null},"Field"),(0,a.yg)("th",{parentName:"tr",align:null},"Default"),(0,a.yg)("th",{parentName:"tr",align:null},"Required"),(0,a.yg)("th",{parentName:"tr",align:null},"Description"))),(0,a.yg)("tbody",{parentName:"table"},(0,a.yg)("tr",{parentName:"tbody"},(0,a.yg)("td",{parentName:"tr",align:null},"name"),(0,a.yg)("td",{parentName:"tr",align:null}),(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"true")),(0,a.yg)("td",{parentName:"tr",align:null},"The name for the collection of template MCPs.")),(0,a.yg)("tr",{parentName:"tbody"},(0,a.yg)("td",{parentName:"tr",align:null},"version"),(0,a.yg)("td",{parentName:"tr",align:null}),(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"true")),(0,a.yg)("td",{parentName:"tr",align:null},"A string version for the collection of template MCPs.")),(0,a.yg)("tr",{parentName:"tbody"},(0,a.yg)("td",{parentName:"tr",align:null},"force"),(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"false")),(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"false")),(0,a.yg)("td",{parentName:"tr",align:null},"Ignores the previous run history, will not skip execution if run previously.")),(0,a.yg)("tr",{parentName:"tbody"},(0,a.yg)("td",{parentName:"tr",align:null},"blocking"),(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"false")),(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"false")),(0,a.yg)("td",{parentName:"tr",align:null},"Run before GMS and other components during upgrade/install if running in split blocking/non-blocking mode.")),(0,a.yg)("tr",{parentName:"tbody"},(0,a.yg)("td",{parentName:"tr",align:null},"async"),(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"true")),(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"false")),(0,a.yg)("td",{parentName:"tr",align:null},"Controls whether the MCPs are executed for sync or async ingestion.")),(0,a.yg)("tr",{parentName:"tbody"},(0,a.yg)("td",{parentName:"tr",align:null},"optional"),(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"false")),(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"false")),(0,a.yg)("td",{parentName:"tr",align:null},"Whether to ignore a failure or fail the entire ",(0,a.yg)("inlineCode",{parentName:"td"},"system-update")," job.")),(0,a.yg)("tr",{parentName:"tbody"},(0,a.yg)("td",{parentName:"tr",align:null},"mcps_location"),(0,a.yg)("td",{parentName:"tr",align:null}),(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"true")),(0,a.yg)("td",{parentName:"tr",align:null},"The location of the file which contains the template MCPs")),(0,a.yg)("tr",{parentName:"tbody"},(0,a.yg)("td",{parentName:"tr",align:null},"values_env"),(0,a.yg)("td",{parentName:"tr",align:null}),(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"false")),(0,a.yg)("td",{parentName:"tr",align:null},"The environment variable which contains override template values.")))),(0,a.yg)("h2",{id:"template-mcps"},"Template MCPs"),(0,a.yg)("p",null,"Template MCPs are stored in a yaml file which uses the mustache templating library to populate values from an optional environment\nvariable. Defaults can be provided inline making override only necessary when providing install/upgrade time configuration."),(0,a.yg)("p",null,"In general the file contains a list of MCPs which follow the schema definition for MCPs exactly. Any valid field for an MCP\nis accepted, including optional fields such as ",(0,a.yg)("inlineCode",{parentName:"p"},"headers"),"."),(0,a.yg)("h3",{id:"example-native-group"},"Example: Native Group"),(0,a.yg)("p",null,"An example template MCP collection, configuration, and values environment variable is shown below which would create a native group."),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},"- entityUrn: urn:li:corpGroup:{{group.id}}\n  entityType: corpGroup\n  aspectName: corpGroupInfo\n  changeType: UPSERT\n  aspect:\n   description: {{group.description}}{{^group.description}}Default description{{/group.description}}\n   displayName: {{group.displayName}}\n   created: {{&auditStamp}}\n   members: [] # required as part of the aspect's schema definition\n   groups: [] # required as part of the aspect's schema definition\n   admins: [] # required as part of the aspect's schema definition\n- entityUrn: urn:li:corpGroup:{{group.id}}\n  entityType: corpGroup\n  aspectName: origin\n  changeType: UPSERT\n  aspect:\n     type: NATIVE\n")),(0,a.yg)("p",null,"Creating an entry in the ",(0,a.yg)("inlineCode",{parentName:"p"},"bootstrap_mcps.yaml")," to populate the values from the environment variable ",(0,a.yg)("inlineCode",{parentName:"p"},"DATAHUB_TEST_GROUP_VALUES")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},'- name: test-group\n  version: v1\n  mcps_location: "bootstrap_mcps/test-group.yaml"\n  values_env: "DATAHUB_TEST_GROUP_VALUES"\n')),(0,a.yg)("p",null,"An example json values are loaded from environment variable in ",(0,a.yg)("inlineCode",{parentName:"p"},"DATAHUB_TEST_GROUP_VALUES")," might look like the following."),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "group": {\n    "id": "mygroup",\n    "displayName": "My Group",\n    "description": "Description of the group"\n  }\n}\n')),(0,a.yg)("p",null,"Using standard mustache template semantics the values in the environment would be inserted into the yaml structure\nand ingested when the ",(0,a.yg)("inlineCode",{parentName:"p"},"system-update")," runs."),(0,a.yg)("h4",{id:"default-values"},"Default values"),(0,a.yg)("p",null,"In the example above, the group's ",(0,a.yg)("inlineCode",{parentName:"p"},"description")," if not provided would default to ",(0,a.yg)("inlineCode",{parentName:"p"},"Default description")," if not specified\nin the values contain in the environment variable override following the standard mustache template semantics."),(0,a.yg)("h4",{id:"auditstamp"},"AuditStamp"),(0,a.yg)("p",null,"A special template reference, ",(0,a.yg)("inlineCode",{parentName:"p"},"{{&auditStamp}}")," can be used to inject an ",(0,a.yg)("inlineCode",{parentName:"p"},"auditStamp")," into the aspect. This can be used to\npopulate required fields of type ",(0,a.yg)("inlineCode",{parentName:"p"},"auditStamp")," calculated from when the MCP is applied. This will insert an inline json representation\nof the ",(0,a.yg)("inlineCode",{parentName:"p"},"auditStamp")," into the location and avoid escaping html characters per standard mustache template indicated by the ",(0,a.yg)("inlineCode",{parentName:"p"},"&")," character."),(0,a.yg)("h3",{id:"ingestion-template-mcps"},"Ingestion Template MCPs"),(0,a.yg)("p",null,"Ingestion template MCPs are slightly more complicated since the ingestion ",(0,a.yg)("inlineCode",{parentName:"p"},"recipe")," is stored as a json string within the aspect.\nFor ingestion recipes, special handling was added so that they can be described naturally in yaml instead of the normally encoded json string."),(0,a.yg)("p",null,"This means that in the example below, the structure beneath the ",(0,a.yg)("inlineCode",{parentName:"p"},"aspect.config.recipe")," path will be automatically converted\nto the required json structure and stored as a string."),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},'- entityType: dataHubIngestionSource\n  entityUrn: urn:li:dataHubIngestionSource:demo-data\n  aspectName: dataHubIngestionSourceInfo\n  changeType: UPSERT\n  aspect:\n    type: "demo-data"\n    name: "demo-data"\n    config:\n      recipe:\n        source:\n          type: "datahub-gc"\n          config: {}\n      executorId: default\n')),(0,a.yg)("h2",{id:"bootstrap_mcpsyaml-override"},(0,a.yg)("inlineCode",{parentName:"h2"},"bootstrap_mcps.yaml")," Override"),(0,a.yg)("p",null,"Additionally, the ",(0,a.yg)("inlineCode",{parentName:"p"},"bootstrap_mcps.yaml")," can be overridden.\nThis might be useful for applying changes to the version when using helm defined template values."),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},"bootstrap:\n  templates:\n    - name: myMCPTemplate\n      version: v1\n      mcps_location: <classpath or file location>\n      values_env: <value environment variable>\n      revision_env: REVISION_ENV\n")),(0,a.yg)("p",null,"In the above example, we've added a ",(0,a.yg)("inlineCode",{parentName:"p"},"revision_env")," which allows overriding the MCP bootstrap definition itself (excluding ",(0,a.yg)("inlineCode",{parentName:"p"},"revision_env"),")."),(0,a.yg)("p",null,"In this example we could configure ",(0,a.yg)("inlineCode",{parentName:"p"},"REVISION_ENV")," to contain a timestamp or hash: ",(0,a.yg)("inlineCode",{parentName:"p"},'{"version":"2024060600"}'),"\nThis value can be changed/incremented each time the helm supplied template values change. This ensures the MCP is updated\nwith the latest values during deployment."),(0,a.yg)("h2",{id:"known-limitations"},"Known Limitations"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"Supported change types:",(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"UPSERT"),(0,a.yg)("li",{parentName:"ul"},"CREATE"),(0,a.yg)("li",{parentName:"ul"},"CREATE_ENTITY")))))}g.isMDXComponent=!0}}]);