mirror of
https://github.com/datahub-project/datahub.git
synced 2025-08-31 04:25:29 +00:00
1 line
50 KiB
JavaScript
1 line
50 KiB
JavaScript
"use strict";(self.webpackChunkdocs_website=self.webpackChunkdocs_website||[]).push([[39425],{15680:(e,n,t)=>{t.d(n,{xA:()=>d,yg:()=>y});var a=t(96540);function s(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function i(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);n&&(a=a.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,a)}return t}function r(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?i(Object(t),!0).forEach((function(n){s(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):i(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function o(e,n){if(null==e)return{};var t,a,s=function(e,n){if(null==e)return{};var t,a,s={},i=Object.keys(e);for(a=0;a<i.length;a++)t=i[a],n.indexOf(t)>=0||(s[t]=e[t]);return s}(e,n);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)t=i[a],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(s[t]=e[t])}return s}var l=a.createContext({}),c=function(e){var n=a.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):r(r({},n),e)),t},d=function(e){var n=c(e.components);return a.createElement(l.Provider,{value:n},e.children)},u="mdxType",p={inlineCode:"code",wrapper:function(e){var n=e.children;return a.createElement(a.Fragment,{},n)}},m=a.forwardRef((function(e,n){var t=e.components,s=e.mdxType,i=e.originalType,l=e.parentName,d=o(e,["components","mdxType","originalType","parentName"]),u=c(t),m=s,y=u["".concat(l,".").concat(m)]||u[m]||p[m]||i;return t?a.createElement(y,r(r({ref:n},d),{},{components:t})):a.createElement(y,r({ref:n},d))}));function y(e,n){var t=arguments,s=n&&n.mdxType;if("string"==typeof e||s){var i=t.length,r=new Array(i);r[0]=m;var o={};for(var l in n)hasOwnProperty.call(n,l)&&(o[l]=n[l]);o.originalType=e,o[u]="string"==typeof e?e:s,r[1]=o;for(var c=2;c<i;c++)r[c]=t[c];return a.createElement.apply(null,r)}return a.createElement.apply(null,t)}m.displayName="MDXCreateElement"},42947:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>d,contentTitle:()=>l,default:()=>y,frontMatter:()=>o,metadata:()=>c,toc:()=>u});t(96540);var a=t(15680);function s(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function i(e,n){return n=null!=n?n:{},Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):function(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);n&&(a=a.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,a)}return t}(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))})),e}function r(e,n){if(null==e)return{};var t,a,s=function(e,n){if(null==e)return{};var t,a,s={},i=Object.keys(e);for(a=0;a<i.length;a++)t=i[a],n.indexOf(t)>=0||(s[t]=e[t]);return s}(e,n);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)t=i[a],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(s[t]=e[t])}return s}const o={sidebar_position:5,title:"DataProcessInstance",slug:"/generated/metamodel/entities/dataprocessinstance",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/generated/metamodel/entities/dataProcessInstance.md"},l="DataProcessInstance",c={unversionedId:"docs/generated/metamodel/entities/dataProcessInstance",id:"docs/generated/metamodel/entities/dataProcessInstance",title:"DataProcessInstance",description:"DataProcessInstance represents an instance of a datajob/jobflow run",source:"@site/genDocs/docs/generated/metamodel/entities/dataProcessInstance.md",sourceDirName:"docs/generated/metamodel/entities",slug:"/generated/metamodel/entities/dataprocessinstance",permalink:"/docs/generated/metamodel/entities/dataprocessinstance",draft:!1,editUrl:"https://github.com/datahub-project/datahub/blob/master/docs/generated/metamodel/entities/dataProcessInstance.md",tags:[],version:"current",sidebarPosition:5,frontMatter:{sidebar_position:5,title:"DataProcessInstance",slug:"/generated/metamodel/entities/dataprocessinstance",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/generated/metamodel/entities/dataProcessInstance.md"},sidebar:"overviewSidebar",previous:{title:"DataProcess",permalink:"/docs/generated/metamodel/entities/dataprocess"},next:{title:"Chart",permalink:"/docs/generated/metamodel/entities/chart"}},d={},u=[{value:"Aspects",id:"aspects",level:2},{value:"dataProcessInstanceInput",id:"dataprocessinstanceinput",level:3},{value:"dataProcessInstanceOutput",id:"dataprocessinstanceoutput",level:3},{value:"dataProcessInstanceProperties",id:"dataprocessinstanceproperties",level:3},{value:"dataProcessInstanceRelationships",id:"dataprocessinstancerelationships",level:3},{value:"status",id:"status",level:3},{value:"testResults",id:"testresults",level:3},{value:"dataPlatformInstance",id:"dataplatforminstance",level:3},{value:"subTypes",id:"subtypes",level:3},{value:"container",id:"container",level:3},{value:"mlTrainingRunProperties",id:"mltrainingrunproperties",level:3},{value:"dataProcessInstanceRunEvent (Timeseries)",id:"dataprocessinstancerunevent-timeseries",level:3},{value:"Relationships",id:"relationships",level:2},{value:"Self",id:"self",level:3},{value:"Outgoing",id:"outgoing",level:3},{value:"Global Metadata Model",id:"global-metadata-model",level:2}],p={toc:u},m="wrapper";function y(e){var{components:n}=e,t=r(e,["components"]);return(0,a.yg)(m,i(function(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{},a=Object.keys(t);"function"==typeof Object.getOwnPropertySymbols&&(a=a.concat(Object.getOwnPropertySymbols(t).filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable})))),a.forEach((function(n){s(e,n,t[n])}))}return e}({},p,t),{components:n,mdxType:"MDXLayout"}),(0,a.yg)("h1",{id:"dataprocessinstance"},"DataProcessInstance"),(0,a.yg)("p",null,"DataProcessInstance represents an instance of a datajob/jobflow run"),(0,a.yg)("h2",{id:"aspects"},"Aspects"),(0,a.yg)("h3",{id:"dataprocessinstanceinput"},"dataProcessInstanceInput"),(0,a.yg)("p",null,"Information about the inputs datasets of a Data process"),(0,a.yg)("details",null,(0,a.yg)("summary",null,"Schema"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "type": "record",\n "Aspect": {\n "name": "dataProcessInstanceInput"\n },\n "name": "DataProcessInstanceInput",\n "namespace": "com.linkedin.dataprocess",\n "fields": [\n {\n "Relationship": {\n "/*": {\n "entityTypes": [\n "dataset",\n "mlModel"\n ],\n "name": "Consumes"\n }\n },\n "Searchable": {\n "/*": {\n "addToFilters": true,\n "fieldName": "inputs",\n "fieldType": "URN",\n "numValuesFieldName": "numInputs",\n "queryByDefault": false\n }\n },\n "type": {\n "type": "array",\n "items": "string"\n },\n "name": "inputs",\n "doc": "Input assets consumed"\n },\n {\n "Relationship": {\n "/*/destinationUrn": {\n "createdActor": "inputEdges/*/created/actor",\n "createdOn": "inputEdges/*/created/time",\n "entityTypes": [\n "dataset",\n "mlModel",\n "dataProcessInstance"\n ],\n "isLineage": true,\n "name": "DataProcessInstanceConsumes",\n "properties": "inputEdges/*/properties",\n "updatedActor": "inputEdges/*/lastModified/actor",\n "updatedOn": "inputEdges/*/lastModified/time"\n }\n },\n "type": [\n "null",\n {\n "type": "array",\n "items": {\n "type": "record",\n "name": "Edge",\n "namespace": "com.linkedin.common",\n "fields": [\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": [\n "null",\n "string"\n ],\n "name": "sourceUrn",\n "default": null,\n "doc": "Urn of the source of this relationship edge.\\nIf not specified, assumed to be the entity that this aspect belongs to."\n },\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": "string",\n "name": "destinationUrn",\n "doc": "Urn of the destination of this relationship edge."\n },\n {\n "type": [\n "null",\n {\n "type": "record",\n "name": "AuditStamp",\n "namespace": "com.linkedin.common",\n "fields": [\n {\n "type": "long",\n "name": "time",\n "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."\n },\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": "string",\n "name": "actor",\n "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change."\n },\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": [\n "null",\n "string"\n ],\n "name": "impersonator",\n "default": null,\n "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor."\n },\n {\n "type": [\n "null",\n "string"\n ],\n "name": "message",\n "default": null,\n "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."\n }\n ],\n "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."\n }\n ],\n "name": "created",\n "default": null,\n "doc": "Audit stamp containing who created this relationship edge and when"\n },\n {\n "type": [\n "null",\n "com.linkedin.common.AuditStamp"\n ],\n "name": "lastModified",\n "default": null,\n "doc": "Audit stamp containing who last modified this relationship edge and when"\n },\n {\n "type": [\n "null",\n {\n "type": "map",\n "values": "string"\n }\n ],\n "name": "properties",\n "default": null,\n "doc": "A generic properties bag that allows us to store specific information on this graph edge."\n }\n ],\n "doc": "A common structure to represent all edges to entities when used inside aspects as collections\\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."\n }\n }\n ],\n "name": "inputEdges",\n "default": null,\n "doc": "Input assets consumed by the data process instance, with additional metadata.\\nCounts as lineage.\\nWill eventually deprecate the inputs field."\n }\n ],\n "doc": "Information about the inputs datasets of a Data process"\n}\n'))),(0,a.yg)("h3",{id:"dataprocessinstanceoutput"},"dataProcessInstanceOutput"),(0,a.yg)("p",null,"Information about the outputs of a Data process"),(0,a.yg)("details",null,(0,a.yg)("summary",null,"Schema"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "type": "record",\n "Aspect": {\n "name": "dataProcessInstanceOutput"\n },\n "name": "DataProcessInstanceOutput",\n "namespace": "com.linkedin.dataprocess",\n "fields": [\n {\n "Relationship": {\n "/*": {\n "entityTypes": [\n "dataset",\n "mlModel"\n ],\n "name": "Produces"\n }\n },\n "Searchable": {\n "/*": {\n "addToFilters": true,\n "fieldName": "outputs",\n "fieldType": "URN",\n "numValuesFieldName": "numOutputs",\n "queryByDefault": false\n }\n },\n "type": {\n "type": "array",\n "items": "string"\n },\n "name": "outputs",\n "doc": "Output assets produced"\n },\n {\n "Relationship": {\n "/*/destinationUrn": {\n "createdActor": "outputEdges/*/created/actor",\n "createdOn": "outputEdges/*/created/time",\n "entityTypes": [\n "dataset",\n "mlModel",\n "dataProcessInstance"\n ],\n "isLineage": true,\n "isUpstream": false,\n "name": "DataProcessInstanceProduces",\n "properties": "outputEdges/*/properties",\n "updatedActor": "outputEdges/*/lastModified/actor",\n "updatedOn": "outputEdges/*/lastModified/time"\n }\n },\n "type": [\n "null",\n {\n "type": "array",\n "items": {\n "type": "record",\n "name": "Edge",\n "namespace": "com.linkedin.common",\n "fields": [\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": [\n "null",\n "string"\n ],\n "name": "sourceUrn",\n "default": null,\n "doc": "Urn of the source of this relationship edge.\\nIf not specified, assumed to be the entity that this aspect belongs to."\n },\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": "string",\n "name": "destinationUrn",\n "doc": "Urn of the destination of this relationship edge."\n },\n {\n "type": [\n "null",\n {\n "type": "record",\n "name": "AuditStamp",\n "namespace": "com.linkedin.common",\n "fields": [\n {\n "type": "long",\n "name": "time",\n "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."\n },\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": "string",\n "name": "actor",\n "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change."\n },\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": [\n "null",\n "string"\n ],\n "name": "impersonator",\n "default": null,\n "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor."\n },\n {\n "type": [\n "null",\n "string"\n ],\n "name": "message",\n "default": null,\n "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."\n }\n ],\n "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."\n }\n ],\n "name": "created",\n "default": null,\n "doc": "Audit stamp containing who created this relationship edge and when"\n },\n {\n "type": [\n "null",\n "com.linkedin.common.AuditStamp"\n ],\n "name": "lastModified",\n "default": null,\n "doc": "Audit stamp containing who last modified this relationship edge and when"\n },\n {\n "type": [\n "null",\n {\n "type": "map",\n "values": "string"\n }\n ],\n "name": "properties",\n "default": null,\n "doc": "A generic properties bag that allows us to store specific information on this graph edge."\n }\n ],\n "doc": "A common structure to represent all edges to entities when used inside aspects as collections\\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."\n }\n }\n ],\n "name": "outputEdges",\n "default": null,\n "doc": "Output assets produced by the data process instance during processing, with additional metadata.\\nCounts as lineage.\\nWill eventually deprecate the outputs field."\n }\n ],\n "doc": "Information about the outputs of a Data process"\n}\n'))),(0,a.yg)("h3",{id:"dataprocessinstanceproperties"},"dataProcessInstanceProperties"),(0,a.yg)("p",null,"The inputs and outputs of this data process"),(0,a.yg)("details",null,(0,a.yg)("summary",null,"Schema"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "type": "record",\n "Aspect": {\n "name": "dataProcessInstanceProperties"\n },\n "name": "DataProcessInstanceProperties",\n "namespace": "com.linkedin.dataprocess",\n "fields": [\n {\n "Searchable": {\n "/*": {\n "fieldType": "TEXT",\n "queryByDefault": true\n }\n },\n "type": {\n "type": "map",\n "values": "string"\n },\n "name": "customProperties",\n "default": {},\n "doc": "Custom property bag."\n },\n {\n "Searchable": {\n "fieldType": "KEYWORD"\n },\n "java": {\n "class": "com.linkedin.common.url.Url",\n "coercerClass": "com.linkedin.common.url.UrlCoercer"\n },\n "type": [\n "null",\n "string"\n ],\n "name": "externalUrl",\n "default": null,\n "doc": "URL where the reference exist"\n },\n {\n "Searchable": {\n "boostScore": 10.0,\n "enableAutocomplete": true,\n "fieldType": "WORD_GRAM"\n },\n "type": "string",\n "name": "name",\n "doc": "Process name"\n },\n {\n "Searchable": {\n "addToFilters": true,\n "fieldName": "processType",\n "fieldType": "KEYWORD",\n "filterNameOverride": "Process Type"\n },\n "type": [\n "null",\n {\n "type": "enum",\n "name": "DataProcessType",\n "namespace": "com.linkedin.dataprocess",\n "symbols": [\n "BATCH_SCHEDULED",\n "BATCH_AD_HOC",\n "STREAMING"\n ]\n }\n ],\n "name": "type",\n "default": null,\n "doc": "Process type"\n },\n {\n "Searchable": {\n "/time": {\n "fieldName": "created",\n "fieldType": "COUNT",\n "queryByDefault": false\n }\n },\n "type": {\n "type": "record",\n "name": "AuditStamp",\n "namespace": "com.linkedin.common",\n "fields": [\n {\n "type": "long",\n "name": "time",\n "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."\n },\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": "string",\n "name": "actor",\n "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change."\n },\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": [\n "null",\n "string"\n ],\n "name": "impersonator",\n "default": null,\n "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor."\n },\n {\n "type": [\n "null",\n "string"\n ],\n "name": "message",\n "default": null,\n "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."\n }\n ],\n "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."\n },\n "name": "created",\n "doc": "Audit stamp containing who reported the lineage and when"\n }\n ],\n "doc": "The inputs and outputs of this data process"\n}\n'))),(0,a.yg)("h3",{id:"dataprocessinstancerelationships"},"dataProcessInstanceRelationships"),(0,a.yg)("p",null,"Information about Data process relationships"),(0,a.yg)("details",null,(0,a.yg)("summary",null,"Schema"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "type": "record",\n "Aspect": {\n "name": "dataProcessInstanceRelationships"\n },\n "name": "DataProcessInstanceRelationships",\n "namespace": "com.linkedin.dataprocess",\n "fields": [\n {\n "Relationship": {\n "entityTypes": [\n "dataJob",\n "dataFlow",\n "dataset"\n ],\n "name": "InstanceOf"\n },\n "Searchable": {\n "/*": {\n "fieldName": "parentTemplate",\n "fieldType": "URN",\n "queryByDefault": false\n }\n },\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": [\n "null",\n "string"\n ],\n "name": "parentTemplate",\n "default": null,\n "doc": "The parent entity whose run instance it is"\n },\n {\n "Relationship": {\n "entityTypes": [\n "dataProcessInstance"\n ],\n "name": "ChildOf"\n },\n "Searchable": {\n "/*": {\n "fieldName": "parentInstance",\n "fieldType": "URN",\n "queryByDefault": false\n }\n },\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": [\n "null",\n "string"\n ],\n "name": "parentInstance",\n "default": null,\n "doc": "The parent DataProcessInstance where it belongs to.\\nIf it is a Airflow Task then it should belong to an Airflow Dag run as well\\nwhich will be another DataProcessInstance"\n },\n {\n "Relationship": {\n "/*": {\n "entityTypes": [\n "dataProcessInstance"\n ],\n "name": "UpstreamOf"\n }\n },\n "Searchable": {\n "/*": {\n "fieldName": "upstream",\n "fieldType": "URN",\n "numValuesFieldName": "numUpstreams",\n "queryByDefault": false\n }\n },\n "type": {\n "type": "array",\n "items": "string"\n },\n "name": "upstreamInstances",\n "doc": "Input DataProcessInstance which triggered this dataprocess instance"\n }\n ],\n "doc": "Information about Data process relationships"\n}\n'))),(0,a.yg)("h3",{id:"status"},"status"),(0,a.yg)("p",null,"The lifecycle status metadata of an entity, e.g. dataset, metric, feature, etc.\nThis aspect is used to represent soft deletes conventionally."),(0,a.yg)("details",null,(0,a.yg)("summary",null,"Schema"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "type": "record",\n "Aspect": {\n "name": "status"\n },\n "name": "Status",\n "namespace": "com.linkedin.common",\n "fields": [\n {\n "Searchable": {\n "fieldType": "BOOLEAN"\n },\n "type": "boolean",\n "name": "removed",\n "default": false,\n "doc": "Whether the entity has been removed (soft-deleted)."\n }\n ],\n "doc": "The lifecycle status metadata of an entity, e.g. dataset, metric, feature, etc.\\nThis aspect is used to represent soft deletes conventionally."\n}\n'))),(0,a.yg)("h3",{id:"testresults"},"testResults"),(0,a.yg)("p",null,"Information about a Test Result"),(0,a.yg)("details",null,(0,a.yg)("summary",null,"Schema"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "type": "record",\n "Aspect": {\n "name": "testResults"\n },\n "name": "TestResults",\n "namespace": "com.linkedin.test",\n "fields": [\n {\n "Relationship": {\n "/*/test": {\n "entityTypes": [\n "test"\n ],\n "name": "IsFailing"\n }\n },\n "Searchable": {\n "/*/test": {\n "fieldName": "failingTests",\n "fieldType": "URN",\n "hasValuesFieldName": "hasFailingTests",\n "queryByDefault": false\n }\n },\n "type": {\n "type": "array",\n "items": {\n "type": "record",\n "name": "TestResult",\n "namespace": "com.linkedin.test",\n "fields": [\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": "string",\n "name": "test",\n "doc": "The urn of the test"\n },\n {\n "type": {\n "type": "enum",\n "symbolDocs": {\n "FAILURE": " The Test Failed",\n "SUCCESS": " The Test Succeeded"\n },\n "name": "TestResultType",\n "namespace": "com.linkedin.test",\n "symbols": [\n "SUCCESS",\n "FAILURE"\n ]\n },\n "name": "type",\n "doc": "The type of the result"\n },\n {\n "type": [\n "null",\n "string"\n ],\n "name": "testDefinitionMd5",\n "default": null,\n "doc": "The md5 of the test definition that was used to compute this result.\\nSee TestInfo.testDefinition.md5 for more information."\n },\n {\n "type": [\n "null",\n {\n "type": "record",\n "name": "AuditStamp",\n "namespace": "com.linkedin.common",\n "fields": [\n {\n "type": "long",\n "name": "time",\n "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."\n },\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": "string",\n "name": "actor",\n "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change."\n },\n {\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": [\n "null",\n "string"\n ],\n "name": "impersonator",\n "default": null,\n "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor."\n },\n {\n "type": [\n "null",\n "string"\n ],\n "name": "message",\n "default": null,\n "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."\n }\n ],\n "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."\n }\n ],\n "name": "lastComputed",\n "default": null,\n "doc": "The audit stamp of when the result was computed, including the actor who computed it."\n }\n ],\n "doc": "Information about a Test Result"\n }\n },\n "name": "failing",\n "doc": "Results that are failing"\n },\n {\n "Relationship": {\n "/*/test": {\n "entityTypes": [\n "test"\n ],\n "name": "IsPassing"\n }\n },\n "Searchable": {\n "/*/test": {\n "fieldName": "passingTests",\n "fieldType": "URN",\n "hasValuesFieldName": "hasPassingTests",\n "queryByDefault": false\n }\n },\n "type": {\n "type": "array",\n "items": "com.linkedin.test.TestResult"\n },\n "name": "passing",\n "doc": "Results that are passing"\n }\n ],\n "doc": "Information about a Test Result"\n}\n'))),(0,a.yg)("h3",{id:"dataplatforminstance"},"dataPlatformInstance"),(0,a.yg)("p",null,"The specific instance of the data platform that this entity belongs to"),(0,a.yg)("details",null,(0,a.yg)("summary",null,"Schema"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "type": "record",\n "Aspect": {\n "name": "dataPlatformInstance"\n },\n "name": "DataPlatformInstance",\n "namespace": "com.linkedin.common",\n "fields": [\n {\n "Searchable": {\n "addToFilters": true,\n "fieldType": "URN",\n "filterNameOverride": "Platform"\n },\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": "string",\n "name": "platform",\n "doc": "Data Platform"\n },\n {\n "Searchable": {\n "addToFilters": true,\n "fieldName": "platformInstance",\n "fieldType": "URN",\n "filterNameOverride": "Platform Instance"\n },\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": [\n "null",\n "string"\n ],\n "name": "instance",\n "default": null,\n "doc": "Instance of the data platform (e.g. db instance)"\n }\n ],\n "doc": "The specific instance of the data platform that this entity belongs to"\n}\n'))),(0,a.yg)("h3",{id:"subtypes"},"subTypes"),(0,a.yg)("p",null,"Sub Types. Use this aspect to specialize a generic Entity\ne.g. Making a Dataset also be a View or also be a LookerExplore"),(0,a.yg)("details",null,(0,a.yg)("summary",null,"Schema"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "type": "record",\n "Aspect": {\n "name": "subTypes"\n },\n "name": "SubTypes",\n "namespace": "com.linkedin.common",\n "fields": [\n {\n "Searchable": {\n "/*": {\n "addToFilters": true,\n "fieldType": "KEYWORD",\n "filterNameOverride": "Sub Type",\n "queryByDefault": false\n }\n },\n "type": {\n "type": "array",\n "items": "string"\n },\n "name": "typeNames",\n "doc": "The names of the specific types."\n }\n ],\n "doc": "Sub Types. Use this aspect to specialize a generic Entity\\ne.g. Making a Dataset also be a View or also be a LookerExplore"\n}\n'))),(0,a.yg)("h3",{id:"container"},"container"),(0,a.yg)("p",null,"Link from an asset to its parent container"),(0,a.yg)("details",null,(0,a.yg)("summary",null,"Schema"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "type": "record",\n "Aspect": {\n "name": "container"\n },\n "name": "Container",\n "namespace": "com.linkedin.container",\n "fields": [\n {\n "Relationship": {\n "entityTypes": [\n "container"\n ],\n "name": "IsPartOf"\n },\n "Searchable": {\n "addToFilters": true,\n "fieldName": "container",\n "fieldType": "URN",\n "filterNameOverride": "Container",\n "hasValuesFieldName": "hasContainer"\n },\n "java": {\n "class": "com.linkedin.common.urn.Urn"\n },\n "type": "string",\n "name": "container",\n "doc": "The parent container of an asset"\n }\n ],\n "doc": "Link from an asset to its parent container"\n}\n'))),(0,a.yg)("h3",{id:"mltrainingrunproperties"},"mlTrainingRunProperties"),(0,a.yg)("p",null,"The inputs and outputs of this training run"),(0,a.yg)("details",null,(0,a.yg)("summary",null,"Schema"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "type": "record",\n "Aspect": {\n "name": "mlTrainingRunProperties"\n },\n "name": "MLTrainingRunProperties",\n "namespace": "com.linkedin.ml.metadata",\n "fields": [\n {\n "Searchable": {\n "/*": {\n "fieldType": "TEXT",\n "queryByDefault": true\n }\n },\n "type": {\n "type": "map",\n "values": "string"\n },\n "name": "customProperties",\n "default": {},\n "doc": "Custom property bag."\n },\n {\n "Searchable": {\n "fieldType": "KEYWORD"\n },\n "java": {\n "class": "com.linkedin.common.url.Url",\n "coercerClass": "com.linkedin.common.url.UrlCoercer"\n },\n "type": [\n "null",\n "string"\n ],\n "name": "externalUrl",\n "default": null,\n "doc": "URL where the reference exist"\n },\n {\n "type": [\n "null",\n "string"\n ],\n "name": "id",\n "default": null,\n "doc": "Run Id of the ML Training Run"\n },\n {\n "type": [\n "null",\n {\n "type": "array",\n "items": "string"\n }\n ],\n "name": "outputUrls",\n "default": null,\n "doc": "List of URLs for the Outputs of the ML Training Run"\n },\n {\n "type": [\n "null",\n {\n "type": "array",\n "items": {\n "type": "record",\n "Aspect": {\n "name": "mlHyperParam"\n },\n "name": "MLHyperParam",\n "namespace": "com.linkedin.ml.metadata",\n "fields": [\n {\n "type": "string",\n "name": "name",\n "doc": "Name of the MLHyperParam"\n },\n {\n "type": [\n "null",\n "string"\n ],\n "name": "description",\n "default": null,\n "doc": "Documentation of the MLHyperParam"\n },\n {\n "type": [\n "null",\n "string"\n ],\n "name": "value",\n "default": null,\n "doc": "The value of the MLHyperParam"\n },\n {\n "type": [\n "null",\n "long"\n ],\n "name": "createdAt",\n "default": null,\n "doc": "Date when the MLHyperParam was developed"\n }\n ],\n "doc": "Properties associated with an ML Hyper Param"\n }\n }\n ],\n "name": "hyperParams",\n "default": null,\n "doc": "Hyperparameters of the ML Training Run"\n },\n {\n "type": [\n "null",\n {\n "type": "array",\n "items": {\n "type": "record",\n "Aspect": {\n "name": "mlMetric"\n },\n "name": "MLMetric",\n "namespace": "com.linkedin.ml.metadata",\n "fields": [\n {\n "type": "string",\n "name": "name",\n "doc": "Name of the mlMetric"\n },\n {\n "type": [\n "null",\n "string"\n ],\n "name": "description",\n "default": null,\n "doc": "Documentation of the mlMetric"\n },\n {\n "type": [\n "null",\n "string"\n ],\n "name": "value",\n "default": null,\n "doc": "The value of the mlMetric"\n },\n {\n "type": [\n "null",\n "long"\n ],\n "name": "createdAt",\n "default": null,\n "doc": "Date when the mlMetric was developed"\n }\n ],\n "doc": "Properties associated with an ML Metric"\n }\n }\n ],\n "name": "trainingMetrics",\n "default": null,\n "doc": "Metrics of the ML Training Run"\n }\n ],\n "doc": "The inputs and outputs of this training run"\n}\n'))),(0,a.yg)("h3",{id:"dataprocessinstancerunevent-timeseries"},"dataProcessInstanceRunEvent (Timeseries)"),(0,a.yg)("p",null,"An event representing the current status of data process run.\nDataProcessRunEvent should be used for reporting the status of a dataProcess' run."),(0,a.yg)("details",null,(0,a.yg)("summary",null,"Schema"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "type": "record",\n "Aspect": {\n "name": "dataProcessInstanceRunEvent",\n "type": "timeseries"\n },\n "name": "DataProcessInstanceRunEvent",\n "namespace": "com.linkedin.dataprocess",\n "fields": [\n {\n "type": "long",\n "name": "timestampMillis",\n "doc": "The event timestamp field as epoch at UTC in milli seconds."\n },\n {\n "type": [\n "null",\n {\n "type": "record",\n "name": "TimeWindowSize",\n "namespace": "com.linkedin.timeseries",\n "fields": [\n {\n "type": {\n "type": "enum",\n "name": "CalendarInterval",\n "namespace": "com.linkedin.timeseries",\n "symbols": [\n "SECOND",\n "MINUTE",\n "HOUR",\n "DAY",\n "WEEK",\n "MONTH",\n "QUARTER",\n "YEAR"\n ]\n },\n "name": "unit",\n "doc": "Interval unit such as minute/hour/day etc."\n },\n {\n "type": "int",\n "name": "multiple",\n "default": 1,\n "doc": "How many units. Defaults to 1."\n }\n ],\n "doc": "Defines the size of a time window."\n }\n ],\n "name": "eventGranularity",\n "default": null,\n "doc": "Granularity of the event if applicable"\n },\n {\n "type": [\n {\n "type": "record",\n "name": "PartitionSpec",\n "namespace": "com.linkedin.timeseries",\n "fields": [\n {\n "TimeseriesField": {},\n "type": "string",\n "name": "partition",\n "doc": "A unique id / value for the partition for which statistics were collected,\\ngenerated by applying the key definition to a given row."\n },\n {\n "type": [\n "null",\n {\n "type": "record",\n "name": "TimeWindow",\n "namespace": "com.linkedin.timeseries",\n "fields": [\n {\n "type": "long",\n "name": "startTimeMillis",\n "doc": "Start time as epoch at UTC."\n },\n {\n "type": "com.linkedin.timeseries.TimeWindowSize",\n "name": "length",\n "doc": "The length of the window."\n }\n ]\n }\n ],\n "name": "timePartition",\n "default": null,\n "doc": "Time window of the partition, if we are able to extract it from the partition key."\n },\n {\n "deprecated": true,\n "type": {\n "type": "enum",\n "name": "PartitionType",\n "namespace": "com.linkedin.timeseries",\n "symbols": [\n "FULL_TABLE",\n "QUERY",\n "PARTITION"\n ]\n },\n "name": "type",\n "default": "PARTITION",\n "doc": "Unused!"\n }\n ],\n "doc": "A reference to a specific partition in a dataset."\n },\n "null"\n ],\n "name": "partitionSpec",\n "default": {\n "partition": "FULL_TABLE_SNAPSHOT",\n "type": "FULL_TABLE",\n "timePartition": null\n },\n "doc": "The optional partition specification."\n },\n {\n "type": [\n "null",\n "string"\n ],\n "name": "messageId",\n "default": null,\n "doc": "The optional messageId, if provided serves as a custom user-defined unique identifier for an aspect value."\n },\n {\n "Searchable": {\n "fieldType": "KEYWORD"\n },\n "java": {\n "class": "com.linkedin.common.url.Url",\n "coercerClass": "com.linkedin.common.url.UrlCoercer"\n },\n "type": [\n "null",\n "string"\n ],\n "name": "externalUrl",\n "default": null,\n "doc": "URL where the reference exist"\n },\n {\n "Searchable": {\n "hasValuesFieldName": "hasRunEvents"\n },\n "TimeseriesField": {},\n "type": {\n "type": "enum",\n "symbolDocs": {\n "STARTED": "The status where the Data processing run is in."\n },\n "name": "DataProcessRunStatus",\n "namespace": "com.linkedin.dataprocess",\n "symbols": [\n "STARTED",\n "COMPLETE"\n ]\n },\n "name": "status"\n },\n {\n "type": [\n "null",\n "int"\n ],\n "name": "attempt",\n "default": null,\n "doc": "Return the try number that this Instance Run is in"\n },\n {\n "TimeseriesField": {},\n "type": [\n "null",\n {\n "type": "record",\n "name": "DataProcessInstanceRunResult",\n "namespace": "com.linkedin.dataprocess",\n "fields": [\n {\n "type": {\n "type": "enum",\n "symbolDocs": {\n "FAILURE": " The Run Failed",\n "SKIPPED": " The Run Skipped",\n "SUCCESS": " The Run Succeeded",\n "UP_FOR_RETRY": " The Run Failed and will Retry"\n },\n "name": "RunResultType",\n "namespace": "com.linkedin.dataprocess",\n "symbols": [\n "SUCCESS",\n "FAILURE",\n "SKIPPED",\n "UP_FOR_RETRY"\n ]\n },\n "name": "type",\n "doc": " The final result, e.g. SUCCESS, FAILURE, SKIPPED, or UP_FOR_RETRY."\n },\n {\n "type": "string",\n "name": "nativeResultType",\n "doc": "It identifies the system where the native result comes from like Airflow, Azkaban, etc.."\n }\n ]\n }\n ],\n "name": "result",\n "default": null,\n "doc": "The final result of the Data Processing run."\n },\n {\n "type": [\n "null",\n "long"\n ],\n "name": "durationMillis",\n "default": null,\n "doc": "The duration of the run in milliseconds."\n }\n ],\n "doc": "An event representing the current status of data process run.\\nDataProcessRunEvent should be used for reporting the status of a dataProcess\' run."\n}\n'))),(0,a.yg)("h2",{id:"relationships"},"Relationships"),(0,a.yg)("h3",{id:"self"},"Self"),(0,a.yg)("p",null,"These are the relationships to itself, stored in this entity's aspects"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"DataProcessInstanceConsumes (via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceInput.inputEdges"),")"),(0,a.yg)("li",{parentName:"ul"},"DataProcessInstanceProduces (via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceOutput.outputEdges"),")"),(0,a.yg)("li",{parentName:"ul"},"ChildOf (via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceRelationships.parentInstance"),")"),(0,a.yg)("li",{parentName:"ul"},"UpstreamOf (via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceRelationships.upstreamInstances"),")")),(0,a.yg)("h3",{id:"outgoing"},"Outgoing"),(0,a.yg)("p",null,"These are the relationships stored in this entity's aspects"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"Consumes"),(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"Dataset via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceInput.inputs")),(0,a.yg)("li",{parentName:"ul"},"MlModel via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceInput.inputs")))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"DataProcessInstanceConsumes"),(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"Dataset via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceInput.inputEdges")),(0,a.yg)("li",{parentName:"ul"},"MlModel via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceInput.inputEdges")))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"Produces"),(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"Dataset via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceOutput.outputs")),(0,a.yg)("li",{parentName:"ul"},"MlModel via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceOutput.outputs")))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"DataProcessInstanceProduces"),(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"Dataset via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceOutput.outputEdges")),(0,a.yg)("li",{parentName:"ul"},"MlModel via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceOutput.outputEdges")))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"InstanceOf"),(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"DataJob via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceRelationships.parentTemplate")),(0,a.yg)("li",{parentName:"ul"},"DataFlow via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceRelationships.parentTemplate")),(0,a.yg)("li",{parentName:"ul"},"Dataset via ",(0,a.yg)("inlineCode",{parentName:"li"},"dataProcessInstanceRelationships.parentTemplate")))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"IsFailing"),(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"Test via ",(0,a.yg)("inlineCode",{parentName:"li"},"testResults.failing")))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"IsPassing"),(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"Test via ",(0,a.yg)("inlineCode",{parentName:"li"},"testResults.passing")))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"IsPartOf"),(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"Container via ",(0,a.yg)("inlineCode",{parentName:"li"},"container.container"))))),(0,a.yg)("h2",{id:"global-metadata-model"},(0,a.yg)("a",{parentName:"h2",href:"https://github.com/datahub-project/static-assets/raw/main/imgs/datahub-metadata-model.png"},"Global Metadata Model")),(0,a.yg)("p",null,(0,a.yg)("img",{parentName:"p",src:"https://github.com/datahub-project/static-assets/raw/main/imgs/datahub-metadata-model.png",alt:"Global Graph"})))}y.isMDXComponent=!0}}]); |