"use strict";(self.webpackChunkdocs_website=self.webpackChunkdocs_website||[]).push([[82341],{15680:(e,a,t)=>{t.d(a,{xA:()=>l,yg:()=>g});var n=t(96540);function r(e,a,t){return a in e?Object.defineProperty(e,a,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[a]=t,e}function i(e,a){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);a&&(n=n.filter((function(a){return Object.getOwnPropertyDescriptor(e,a).enumerable}))),t.push.apply(t,n)}return t}function s(e){for(var a=1;a<arguments.length;a++){var t=null!=arguments[a]?arguments[a]:{};a%2?i(Object(t),!0).forEach((function(a){r(e,a,t[a])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):i(Object(t)).forEach((function(a){Object.defineProperty(e,a,Object.getOwnPropertyDescriptor(t,a))}))}return e}function o(e,a){if(null==e)return{};var t,n,r=function(e,a){if(null==e)return{};var t,n,r={},i=Object.keys(e);for(n=0;n<i.length;n++)t=i[n],a.indexOf(t)>=0||(r[t]=e[t]);return r}(e,a);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n<i.length;n++)t=i[n],a.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var c=n.createContext({}),d=function(e){var a=n.useContext(c),t=a;return e&&(t="function"==typeof e?e(a):s(s({},a),e)),t},l=function(e){var a=d(e.components);return n.createElement(c.Provider,{value:a},e.children)},p="mdxType",u={inlineCode:"code",wrapper:function(e){var a=e.children;return n.createElement(n.Fragment,{},a)}},m=n.forwardRef((function(e,a){var t=e.components,r=e.mdxType,i=e.originalType,c=e.parentName,l=o(e,["components","mdxType","originalType","parentName"]),p=d(t),m=r,g=p["".concat(c,".").concat(m)]||p[m]||u[m]||i;return t?n.createElement(g,s(s({ref:a},l),{},{components:t})):n.createElement(g,s({ref:a},l))}));function g(e,a){var t=arguments,r=a&&a.mdxType;if("string"==typeof e||r){var i=t.length,s=new Array(i);s[0]=m;var o={};for(var c in a)hasOwnProperty.call(a,c)&&(o[c]=a[c]);o.originalType=e,o[p]="string"==typeof e?e:r,s[1]=o;for(var d=2;d<i;d++)s[d]=t[d];return n.createElement.apply(null,s)}return n.createElement.apply(null,t)}m.displayName="MDXCreateElement"},11850:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>l,contentTitle:()=>c,default:()=>g,frontMatter:()=>o,metadata:()=>d,toc:()=>p});t(96540);var n=t(15680);function r(e,a,t){return a in e?Object.defineProperty(e,a,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[a]=t,e}function i(e,a){return a=null!=a?a:{},Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):function(e,a){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);a&&(n=n.filter((function(a){return Object.getOwnPropertyDescriptor(e,a).enumerable}))),t.push.apply(t,n)}return t}(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))})),e}function s(e,a){if(null==e)return{};var t,n,r=function(e,a){if(null==e)return{};var t,n,r={},i=Object.keys(e);for(n=0;n<i.length;n++)t=i[n],a.indexOf(t)>=0||(r[t]=e[t]);return r}(e,a);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n<i.length;n++)t=i[n],a.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}const o={title:"API Tracing",slug:"/advanced/api-tracing",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/advanced/api-tracing.md"},c="API Tracing",d={unversionedId:"docs/advanced/api-tracing",id:"version-1.1.0/docs/advanced/api-tracing",title:"API Tracing",description:"Introduction",source:"@site/versioned_docs/version-1.1.0/docs/advanced/api-tracing.md",sourceDirName:"docs/advanced",slug:"/advanced/api-tracing",permalink:"/docs/1.1.0/advanced/api-tracing",draft:!1,editUrl:"https://github.com/datahub-project/datahub/blob/master/docs/advanced/api-tracing.md",tags:[],version:"1.1.0",frontMatter:{title:"API Tracing",slug:"/advanced/api-tracing",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/advanced/api-tracing.md"},sidebar:"overviewSidebar",previous:{title:"Extending the Metadata Model",permalink:"/docs/1.1.0/metadata-modeling/extending-the-metadata-model"},next:{title:"React Analytics",permalink:"/docs/1.1.0/datahub-web-react/src/app/analytics"}},l={},p=[{value:"Introduction",id:"introduction",level:2},{value:"Architecture Overview",id:"architecture-overview",level:2},{value:"Trace API",id:"trace-api",level:2},{value:"Retrieving the <code>trace id</code>",id:"retrieving-the-trace-id",level:3},{value:"Write States",id:"write-states",level:3},{value:"Using the Trace API",id:"using-the-trace-api",level:3},{value:"Ingestion Tracing (Experimental)",id:"ingestion-tracing-experimental",level:3},{value:"Trace Performance",id:"trace-performance",level:2},{value:"Real World Test",id:"real-world-test",level:3},{value:"Trace Exporters",id:"trace-exporters",level:2},{value:"Trace Log Export",id:"trace-log-export",level:3},{value:"Design Notes",id:"design-notes",level:2}],u={toc:p},m="wrapper";function g(e){var{components:a}=e,t=s(e,["components"]);return(0,n.yg)(m,i(function(e){for(var a=1;a<arguments.length;a++){var t=null!=arguments[a]?arguments[a]:{},n=Object.keys(t);"function"==typeof Object.getOwnPropertySymbols&&(n=n.concat(Object.getOwnPropertySymbols(t).filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable})))),n.forEach((function(a){r(e,a,t[a])}))}return e}({},u,t),{components:a,mdxType:"MDXLayout"}),(0,n.yg)("h1",{id:"api-tracing"},"API Tracing"),(0,n.yg)("h2",{id:"introduction"},"Introduction"),(0,n.yg)("p",null,"DataHub's asynchronous APIs enable high-volume data operations, particularly for bulk ingestion processes. While these\nAPIs optimize throughput, they previously lacked built-in validation mechanisms for operation status. Consequently,\ndetecting processing issues required direct monitoring of backend system metrics and logs."),(0,n.yg)("p",null,"To address this limitation, DataHub implemented a trace/request ID system that enables end-to-end tracking of write\noperations. This tracing mechanism is particularly crucial given DataHub's multi-stage write architecture, where data\npropagates through multiple components and persists across distinct storage systems. The trace ID maintains continuity\nthroughout this complex processing pipeline, providing visibility into the operation's status at each stage of execution."),(0,n.yg)("p",null,"The system effectively balances the performance benefits of asynchronous processing with the operational necessity of\nrequest tracking and validation. This enhancement significantly improves observability without compromising the\nthroughput advantages of bulk operations."),(0,n.yg)("h2",{id:"architecture-overview"},"Architecture Overview"),(0,n.yg)("p",null,"Shown below is the write path for an asynchronous write within DataHub. For more information about MCPs please see\nthe documentation on ",(0,n.yg)("a",{parentName:"p",href:"/docs/1.1.0/advanced/mcp-mcl"},"MetadataChangeProposal & MetadataChangeLog Events"),"."),(0,n.yg)("p",{align:"center"},(0,n.yg)("img",{width:"90%",src:"https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/advanced/mcp-mcl/async-ingestion.svg"})),(0,n.yg)("p",null,"A successful write operation requires data persistence in at least one storage system, though typically both primary and\nsearch storage systems must be updated. The storage architecture consists of two main components:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Primary Storage: Comprises MySQL, Postgres, or Cassandra, serving as the persistent store for all non-Timeseries aspects."),(0,n.yg)("li",{parentName:"ul"},"Search Storage: Utilizes either Elasticsearch or OpenSearch systems.")),(0,n.yg)("p",null,"In most operational scenarios, write operations must successfully complete across both storage layers to maintain system\nconsistency and ensure complete data availability."),(0,n.yg)("h2",{id:"trace-api"},"Trace API"),(0,n.yg)("p",null,"Known Limitations:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Tracing can fail in some cases where a batch of aspects introduces a conflict between a system generated aspect and one\nincluded within a batch. An example, ",(0,n.yg)("inlineCode",{parentName:"li"},"upstreamLineage")," and ",(0,n.yg)("inlineCode",{parentName:"li"},"siblings")," aspects together may conflict with a system generated\n",(0,n.yg)("inlineCode",{parentName:"li"},"siblings")," aspect normally generated from ",(0,n.yg)("inlineCode",{parentName:"li"},"upstreamLineage")," in an MCL hook.")),(0,n.yg)("p",null,"The trace API's status retrieval functionality requires three key identifiers to locate specific write operations:\nthe trace ID (unique to the request), the URN, and the aspect name. This combination of identifiers ensures precise\noperation tracking within the system."),(0,n.yg)("p",null,"For batch operations involving multiple URNs and aspects, a single trace ID is assigned to monitor the entire request.\nIn asynchronous mode, the system maintains independent status tracking for each aspect within the batch, allowing for\ngranular operation monitoring."),(0,n.yg)("p",null,"The API returns a comprehensive status report that includes:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Per-aspect success/failure status"),(0,n.yg)("li",{parentName:"ul"},"Detailed status breakdowns for each storage system"),(0,n.yg)("li",{parentName:"ul"},"Write states as defined in the ",(0,n.yg)("a",{parentName:"li",href:"#Write-States"},"Write States")," documentation"),(0,n.yg)("li",{parentName:"ul"},"Error information from MCP processing, when applicable, to facilitate debugging")),(0,n.yg)("p",null,"This structured approach to status reporting enables precise monitoring of complex write operations across the system's\nvarious components."),(0,n.yg)("h3",{id:"retrieving-the-trace-id"},"Retrieving the ",(0,n.yg)("inlineCode",{parentName:"h3"},"trace id")),(0,n.yg)("p",null,"DataHub's asynchronous APIs provide trace ID information through two distinct mechanisms:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"HTTP Response Header: A W3C-compliant ",(0,n.yg)("inlineCode",{parentName:"li"},"traceparent")," header is included in all API responses\nThe complete header value serves as a valid trace ID"),(0,n.yg)("li",{parentName:"ul"},"System Metadata: For OpenAPI v3 APIs and those returning systemMetadata, the trace ID is accessible via the\n",(0,n.yg)("inlineCode",{parentName:"li"},"telemetryTraceId")," property within systemMetadata")),(0,n.yg)("p",null,"While these two trace ID formats differ structurally\u2014with the ",(0,n.yg)("inlineCode",{parentName:"p"},"traceparent")," adhering to W3C's Trace Context\nspecification\u2014both formats are fully compatible with the Trace API for operation tracking purposes."),(0,n.yg)("p",null,"Header Example:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-text"},"traceparent: 00-00062c53a468cbd8077e7dd079846870-9199effb49910b4e-01\n")),(0,n.yg)("p",null,(0,n.yg)("inlineCode",{parentName:"p"},"SystemMetadata")," Example:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-json"},'[\n  {\n    "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)",\n    "status": {\n      "value": {\n        "removed": false\n      },\n      "systemMetadata": {\n        "properties": {\n          "telemetryLog": "false",\n          "telemetryQueueSpanId": "ee9e40edcb66ce4f",\n          "telemetryTraceId": "00062c53a468cbd8077e7dd079846870",\n          "telemetryEnqueuedAt": "1737587612508"\n        }\n      }\n    }\n  }\n]\n')),(0,n.yg)("h3",{id:"write-states"},"Write States"),(0,n.yg)("p",null,"As mentioned earlier, there are multiple states for an aspect write both storage systems. These states are as follows:"),(0,n.yg)("table",null,(0,n.yg)("thead",{parentName:"table"},(0,n.yg)("tr",{parentName:"thead"},(0,n.yg)("th",{parentName:"tr",align:null},"Write State"),(0,n.yg)("th",{parentName:"tr",align:null},"Description"))),(0,n.yg)("tbody",{parentName:"table"},(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"ERROR")),(0,n.yg)("td",{parentName:"tr",align:null},"This state indicates an error occurred when processing the write request.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"PENDING")),(0,n.yg)("td",{parentName:"tr",align:null},"A pending state indicates that the write is queued and the consumer has not yet processed the message.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"ACTIVE_STATE")),(0,n.yg)("td",{parentName:"tr",align:null},"The write was successful and is the current value.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"HISTORIC_STATE")),(0,n.yg)("td",{parentName:"tr",align:null},"The write was successful, however it has been overwritten by a newer value.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"NO_OP")),(0,n.yg)("td",{parentName:"tr",align:null},"The write is not applicable for a given storage system.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"UNKNOWN")),(0,n.yg)("td",{parentName:"tr",align:null},"We are unable to determine the state of the write and no record of its failure exists either.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"TRACE_NOT_IMPLEMENTED")),(0,n.yg)("td",{parentName:"tr",align:null},"We have not yet implemented tracing a particular aspect type. This applies to Timeseries aspects.")))),(0,n.yg)("h3",{id:"using-the-trace-api"},"Using the Trace API"),(0,n.yg)("p",null,"The Trace API is implemented as an OpenAPI endpoint and can be used both programmatically and through the Swagger UI."),(0,n.yg)("p",null,"Required Values:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("inlineCode",{parentName:"li"},"traceId")," - The ",(0,n.yg)("inlineCode",{parentName:"li"},"trace id")," associated with the write request. See the previous ",(0,n.yg)("a",{parentName:"li",href:"#retrieving-the-trace-id"},"Retrieving the ",(0,n.yg)("inlineCode",{parentName:"a"},"trace id"))," section for how to find this id."),(0,n.yg)("li",{parentName:"ul"},"URN/Aspect names - These are passed as a POST body and should represent at least a subset of the URN/aspects from the initial request.\nAn example is shown here for a single URN and 2 aspects ","[",(0,n.yg)("inlineCode",{parentName:"li"},"datasetInfo"),", ",(0,n.yg)("inlineCode",{parentName:"li"},"status"),"]",".",(0,n.yg)("pre",{parentName:"li"},(0,n.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "urn:li:dataset:(urn:li:dataPlatform:bigquery,transactions.user_profile,PROD)": [\n    "datasetInfo",\n    "status"\n  ]\n}\n'))),(0,n.yg)("li",{parentName:"ul"},"Authorization token")),(0,n.yg)("p",null,"Optional Parameters:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("inlineCode",{parentName:"li"},"onlyIncludeErrors")," (default: ",(0,n.yg)("inlineCode",{parentName:"li"},"true"),") - If this parameter is set to ",(0,n.yg)("inlineCode",{parentName:"li"},"true"),", the response will only include status information on the failed aspects."),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("inlineCode",{parentName:"li"},"detailed")," (default: ",(0,n.yg)("inlineCode",{parentName:"li"},"false"),") - If set to ",(0,n.yg)("inlineCode",{parentName:"li"},"true"),", will include detailed information from exceptions for failed MCPs."),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("inlineCode",{parentName:"li"},"skipCache")," (default: ",(0,n.yg)("inlineCode",{parentName:"li"},"false"),") - If set to ",(0,n.yg)("inlineCode",{parentName:"li"},"true"),", will bypass a short-lived cache of the kafka consumer group offsets.")),(0,n.yg)("p",null,"The following shows a few examples of requests/response pairs."),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Successful Write",(0,n.yg)("ul",{parentName:"li"},(0,n.yg)("li",{parentName:"ul"},"Request for URN ",(0,n.yg)("inlineCode",{parentName:"li"},"urn:li:dataset:(urn:li:dataPlatform:bigquery,transactions.user_profile,PROD)")," and aspect ",(0,n.yg)("inlineCode",{parentName:"li"},"status"),(0,n.yg)("pre",{parentName:"li"},(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"curl -v 'http://localhost:8080/openapi/v1/trace/write/00062c2b698bcb28e92508f8f311802d?onlyIncludeErrors=false&detailed=true&skipCache=false' \\\n  -H 'accept: application/json' \\\n  -H 'Content-Type: application/json' \\\n  -H 'Authorization: Bearer <TOKEN>' \\\n  -d '{\n  \"urn:li:dataset:(urn:li:dataPlatform:bigquery,transactions.user_profile,PROD)\": [\n    \"status\"\n  ]\n}' | jq\n"))),(0,n.yg)("li",{parentName:"ul"},"Example response",(0,n.yg)("pre",{parentName:"li"},(0,n.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "urn:li:dataset:(urn:li:dataPlatform:bigquery,transactions.user_profile,PROD)": {\n    "status": {\n      "success": true,\n      "primaryStorage": {\n        "writeStatus": "ACTIVE_STATE"\n      },\n      "searchStorage": {\n        "writeStatus": "ACTIVE_STATE"\n      }\n    }\n  }\n}\n'))))),(0,n.yg)("li",{parentName:"ul"},"Error with exception details",(0,n.yg)("ul",{parentName:"li"},(0,n.yg)("li",{parentName:"ul"},"Example request",(0,n.yg)("pre",{parentName:"li"},(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"curl -v 'http://localhost:8080/openapi/v1/trace/write/00062c543e4550c8400e6f6864471a20?onlyIncludeErrors=true&detailed=true&skipCache=false' \\\n  -H 'accept: application/json' \\\n  -H 'Content-Type: application/json' \\\n  -H 'Authorization: Bearer <TOKEN>' \\\n  -d '{\"urn:li:dataset:(urn:li:dataPlatform:bigquery,transactions.user_profile,PROD)\": [\"status\"]}'\n"))),(0,n.yg)("li",{parentName:"ul"},"Example response",(0,n.yg)("pre",{parentName:"li"},(0,n.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "urn:li:dataset:(urn:li:dataPlatform:bigquery,transactions.user_profile,PROD)": {\n    "status": {\n      "success": false,\n      "primaryStorage": {\n        "writeStatus": "ERROR",\n        "writeExceptions": [\n          {\n            "message": "Expected version -100000, actual version -1",\n            "exceptionClass": "com.linkedin.metadata.aspect.plugins.validation.AspectValidationException",\n            "stackTrace": [\n              "com.linkedin.metadata.aspect.plugins.validation.AspectValidationException.forPrecondition(AspectValidationException.java:33)",\n              "com.linkedin.metadata.aspect.plugins.validation.AspectValidationException.forPrecondition(AspectValidationException.java:25)",\n              "com.linkedin.metadata.aspect.validation.ConditionalWriteValidator.validateVersionPrecondition(ConditionalWriteValidator.java:152)",\n              "com.linkedin.metadata.aspect.validation.ConditionalWriteValidator.lambda$validatePreCommitAspects$2(ConditionalWriteValidator.java:100)",\n              "java.base/java.util.Optional.flatMap(Optional.java:289)",\n              "com.linkedin.metadata.aspect.validation.ConditionalWriteValidator.validatePreCommitAspects(ConditionalWriteValidator.java:98)",\n              "com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator.validatePreCommit(AspectPayloadValidator.java:38)",\n              "com.linkedin.metadata.aspect.batch.AspectsBatch.lambda$validatePreCommit$4(AspectsBatch.java:129)",\n              "java.base/java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:273)",\n              "java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1625)",\n              "java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)",\n              "java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)",\n              "java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)",\n              "java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)",\n              "java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)",\n              "java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:596)",\n              "com.linkedin.metadata.aspect.batch.AspectsBatch.validatePreCommit(AspectsBatch.java:130)"\n            ]\n          }\n        ]\n      },\n      "searchStorage": {\n        "writeStatus": "ERROR",\n        "writeMessage": "Primary storage write failed."\n      }\n    }\n  }\n}\n')))))),(0,n.yg)("h3",{id:"ingestion-tracing-experimental"},"Ingestion Tracing (Experimental)"),(0,n.yg)("p",null,"Known Limitations:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Only OpenAPI is supported"),(0,n.yg)("li",{parentName:"ul"},"ASYNC_WAIT/ASYNC - Async modes are impacted by kafka lag.")),(0,n.yg)("p",null,"Ingestion can be used with tracing enabled however it does require using a ",(0,n.yg)("inlineCode",{parentName:"p"},"ASYNC_WAIT")," emit mode as well as the ",(0,n.yg)("inlineCode",{parentName:"p"},"OPENAPI"),". This\ncan be enabled by setting a couple environment variables shown below."),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"  DATAHUB_REST_SINK_DEFAULT_ENDPOINT=OPENAPI \\\n  DATAHUB_EMIT_MODE=ASYNC_WAIT \\\n  datahub ingest ...\n")),(0,n.yg)("h2",{id:"trace-performance"},"Trace Performance"),(0,n.yg)("p",null,"The Trace API's performance profile varies based on operation status:"),(0,n.yg)("p",null,"Successful Operations:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Optimal performance through direct storage access"),(0,n.yg)("li",{parentName:"ul"},"Requires single lookup operations from SQL and Elasticsearch"),(0,n.yg)("li",{parentName:"ul"},"Bypasses Kafka interaction entirely")),(0,n.yg)("p",null,"Error State Operations:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Performance impact due to required Kafka topic inspection",(0,n.yg)("ul",{parentName:"li"},(0,n.yg)("li",{parentName:"ul"},"Optimization mechanisms implemented:",(0,n.yg)("ul",{parentName:"li"},(0,n.yg)("li",{parentName:"ul"},"Timestamp-based offset seeking for efficient topic traversal"),(0,n.yg)("li",{parentName:"ul"},"Parallel trace processing with controlled concurrency"),(0,n.yg)("li",{parentName:"ul"},"Offset caching system to enhance response times",(0,n.yg)("ul",{parentName:"li"},(0,n.yg)("li",{parentName:"ul"},"Cache bypass available via skipCache parameter when data currency is critical")))))))),(0,n.yg)("p",null,"The performance differential between success and error states stems primarily from the additional overhead of Kafka\ntopic inspection required for error tracking and diagnosis."),(0,n.yg)("p",null,"For more detail, please see the ",(0,n.yg)("a",{parentName:"p",href:"#design-notes"},"Design Notes")," section."),(0,n.yg)("h3",{id:"real-world-test"},"Real World Test"),(0,n.yg)("p",null,"A test was executed with 627,712 aspects in ",(0,n.yg)("inlineCode",{parentName:"p"},"ASYNC_BATCH")," and averaged over 2 runs. The overhead from tracing introduced\na 3.84% increase in runtime. Example test runs shown below."),(0,n.yg)("p",null,"Without Tracing:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "total_records_written": 627712,\n  "records_written_per_second": 376,\n  "total_duration_in_seconds": 1665.25,\n  "mode": "ASYNC_BATCH",\n  "max_threads": 15,\n  "gms_version": "v0.3.9.3",\n  "pending_requests": 0,\n  "async_batches_prepared": 6290,\n  "async_batches_split": 0,\n  "main_thread_blocking_timer": "1186.301 seconds"\n}\n')),(0,n.yg)("p",null,"With Tracing:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "total_records_written": 627612,\n  "records_written_per_second": 368,\n  "total_duration_in_seconds": 1701.51,\n  "mode": "ASYNC_BATCH",\n  "max_threads": 15,\n  "gms_version": "v0.3.9.3",\n  "pending_requests": 0,\n  "async_batches_prepared": 6291,\n  "async_batches_split": 0,\n  "main_thread_blocking_timer": "1224.738 seconds"\n}\n')),(0,n.yg)("h2",{id:"trace-exporters"},"Trace Exporters"),(0,n.yg)("p",null,"At the foundation of the trace instrumentation is OpenTelemetry which has been a part of DataHub for quite some time. As\ndocumented in the ",(0,n.yg)("a",{parentName:"p",href:"/docs/1.1.0/advanced/monitoring"},"Monitoring")," section, OpenTelemetry can be configured to export traces\nto external systems. For the Trace API to function, this external system is NOT required."),(0,n.yg)("h3",{id:"trace-log-export"},"Trace Log Export"),(0,n.yg)("p",null,"A special log-based OpenTelemetry exporter was implemented for debugging purposes. When selectively activated for a given\nrequest it will print ",(0,n.yg)("inlineCode",{parentName:"p"},"trace id"),"s and detailed timing information as the request traverses the different components of DataHub.\nThe output of these logs is also not required for the Trace API to function, however it leverages the same underlying OpenTelemetry\nfoundation."),(0,n.yg)("p",null,"Activating a trace log is done using one of these methods:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"HTTP Header: ",(0,n.yg)("inlineCode",{parentName:"li"},"X-Enable-Trace-Log: true")),(0,n.yg)("li",{parentName:"ul"},"Cookie: ",(0,n.yg)("inlineCode",{parentName:"li"},"enable-trace-log: true"),(0,n.yg)("ul",{parentName:"li"},(0,n.yg)("li",{parentName:"ul"},"javascript: ",(0,n.yg)("inlineCode",{parentName:"li"},'document.cookie = "enable-trace-log=true";'))))),(0,n.yg)("p",null,"Example logs for a single request with tracing logging enabled:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"GMS")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-text"},"i.d.metadata.context.RequestContext:53 - RequestContext{actorUrn='urn:li:corpuser:datahub', sourceIP='172.18.0.5', requestAPI=OPENAPI, requestID='createAspect([dataset])', userAgent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: a2898a18f9f0c4f1, ParentId: dd746f079d1232ba, Name: ingestTimeseriesProposal, Duration: 0.03 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={async=true, batch.size=1}, capacity=128, totalAddedValues=2}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 02e058ff616e4c99, ParentId: 7ed88659811a8fdb, Name: produceMetadataChangeProposal, Duration: 0.03 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={messaging.destination_kind=topic, messaging.system=kafka, messaging.destination=MetadataChangeProposal_v1, messaging.operation=publish, queue.enqueued_at=1737418391958}, capacity=128, totalAddedValues=5}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 7ed88659811a8fdb, ParentId: dd746f079d1232ba, Name: ingestProposalAsync, Duration: 2.57 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1}, capacity=128, totalAddedValues=1}\n")),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"MCE Consumer")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-text"},"c.l.m.k.MetadataChangeProposalsProcessor:89 - Got MCP event key: urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.daily_temperature,PROD), topic: MetadataChangeProposal_v1, partition: 0, offset: 75, value size: 412, timestamp: 1737418391959\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: a65075fe0982d873, ParentId: 02e058ff616e4c99, Name: consume, Duration: 0.01 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={messaging.destination_kind=topic, queue.duration_ms=4, messaging.system=kafka, messaging.destination=MetadataChangeProposal_v1, messaging.operation=receive, queue.enqueued_at=1737418391958}, capacity=128, totalAddedValues=6}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: dd746f079d1232ba, ParentId: 0000000000000000, Name: POST /openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cclimate.daily_temperature%2CPROD%29/status, Duration: 16.18 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={request.api=OPENAPI, http.status_code=202, user.id=urn:li:corpuser:datahub, http.url=/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cclimate.daily_temperature%2CPROD%29/status, request.id=createAspect([dataset]), http.method=POST}, capacity=128, totalAddedValues=6}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 94a019b95154c0e7, ParentId: 0cb378fe4f5ad185, Name: ingestProposalSync, Duration: 0.01 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=0}, capacity=128, totalAddedValues=1}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 0cb378fe4f5ad185, ParentId: 68df6bc4729dc0a2, Name: ingestTimeseriesProposal, Duration: 0.25 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={async=false, batch.size=1}, capacity=128, totalAddedValues=2}\nc.l.m.entity.EntityServiceImpl:988 - Ingesting aspects batch to database: AspectsBatchImpl{items=[ChangeMCP{changeType=UPSERT, urn=urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.daily_temperature,PROD), aspectName='status', recordTemplate={removed=false}, systemMetadata={lastObserved=1737418391954, version=1, properties={telemetryLog=true, telemetryQueueSpanId=02e058ff616e4c99, telemetryEnqueu...}]}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 4754a1c02dadec4c, ParentId: ef383b26f0040fc5, Name: retentionService, Duration: 0.09 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1}, capacity=128, totalAddedValues=1}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: ef383b26f0040fc5, ParentId: 7ae629151400fc18, Name: ingestAspectsToLocalDB, Duration: 18.64 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1, dwizName=com.linkedin.metadata.entity.EntityServiceImpl.ingestAspectsToLocalDB}, capacity=128, totalAddedValues=2}\nc.l.m.entity.EntityServiceImpl:1900 - Producing MCL for ingested aspect status, urn urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.daily_temperature,PROD)\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: f1a8a1da99f1ae23, ParentId: c5f8b3884060722c, Name: produceMetadataChangeLog, Duration: 0.10 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={messaging.destination_kind=topic, messaging.system=kafka, messaging.destination=MetadataChangeLog_Versioned_v1, messaging.operation=publish, queue.enqueued_at=1737418391982}, capacity=128, totalAddedValues=5}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: c5f8b3884060722c, ParentId: 7ae629151400fc18, Name: emitMCL, Duration: 14.32 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1}, capacity=128, totalAddedValues=1}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 7ae629151400fc18, ParentId: 68df6bc4729dc0a2, Name: ingestProposalSync, Duration: 37.90 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1}, capacity=128, totalAddedValues=1}\nc.l.m.k.MetadataChangeProposalsProcessor:128 - Successfully processed MCP event urn: urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.daily_temperature,PROD)\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 68df6bc4729dc0a2, ParentId: 02e058ff616e4c99, Name: consume, Duration: 39.11 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1, dwizName=com.linkedin.metadata.kafka.MetadataChangeProposalsProcessor.consume}, capacity=128, totalAddedValues=2}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 04dc44653b634df2, ParentId: 02e058ff616e4c99, Name: consume, Duration: 0.03 ms\n")),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"MAE Consumer")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-text"},"i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={messaging.destination_kind=topic, queue.duration_ms=22, messaging.system=kafka, messaging.destination=MetadataChangeLog_Versioned_v1, messaging.operation=receive, queue.enqueued_at=1737418391982}, capacity=128, totalAddedValues=6}\nc.l.metadata.kafka.MCLKafkaListener:96 - Invoking MCL hooks for consumer: generic-mae-consumer-job-client urn: urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.daily_temperature,PROD), aspect name: status, entity type: dataset, change type: UPSERT\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 3c3c055c360dc8e4, ParentId: 1de99215a0e82697, Name: FormAssignmentHook, Duration: 0.06 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.FormAssignmentHook_latency}, capacity=128, totalAddedValues=1}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 8e238d0156baacc4, ParentId: 1de99215a0e82697, Name: IngestionSchedulerHook, Duration: 0.05 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.IngestionSchedulerHook_latency}, capacity=128, totalAddedValues=1}\nc.l.m.s.e.update.ESBulkProcessor:85 - Added request id: urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cclimate.daily_temperature%2CPROD%29, operation type: UPDATE, index: datasetindex_v2\nc.l.m.s.e.update.ESBulkProcessor:85 - Added request id: SIHRXj1ktF7qkwPBZO8w0A==, operation type: UPDATE, index: system_metadata_service_v1\nc.l.m.s.e.update.ESBulkProcessor:85 - Added request id: 2p3742l4sFS3wcL82Qh2lQ==, operation type: UPDATE, index: system_metadata_service_v1\nc.l.m.s.e.update.ESBulkProcessor:85 - Added request id: CfZKRLsf25/e3p3mURzlnA==, operation type: UPDATE, index: system_metadata_service_v1\nc.l.m.s.e.update.ESBulkProcessor:85 - Added request id: 8tvhG5ARd5BOdEbqaZkE0g==, operation type: UPDATE, index: system_metadata_service_v1\nc.l.m.s.e.update.ESBulkProcessor:85 - Added request id: rAvQOOBItiKAym622S4dcQ==, operation type: UPDATE, index: system_metadata_service_v1\nc.l.m.s.e.update.ESBulkProcessor:85 - Added request id: YqT6TNy7MAMOAyVXh6abMA==, operation type: UPDATE, index: system_metadata_service_v1\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 054ac726204b449c, ParentId: 1de99215a0e82697, Name: UpdateIndicesHook, Duration: 47.31 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.UpdateIndicesHook_latency}, capacity=128, totalAddedValues=1}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 14d9ded49a94c7b8, ParentId: 1de99215a0e82697, Name: IncidentsSummaryHook, Duration: 0.09 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.IncidentsSummaryHook_latency}, capacity=128, totalAddedValues=1}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: a92d9e54ade6073b, ParentId: 1de99215a0e82697, Name: EntityChangeEventGeneratorHook, Duration: 9.10 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.EntityChangeEventGeneratorHook_latency}, capacity=128, totalAddedValues=1}\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: c06dc7f131e57fca, ParentId: 1de99215a0e82697, Name: SiblingAssociationHook, Duration: 0.07 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.SiblingAssociationHook_latency}, capacity=128, totalAddedValues=1}\nc.l.metadata.kafka.MCLKafkaListener:139 - Successfully completed MCL hooks for consumer: generic-mae-consumer-job-client urn: urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.daily_temperature,PROD)\ni.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 1de99215a0e82697, ParentId: 02e058ff616e4c99, Name: consume, Duration: 58.67 ms\ni.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1, dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.consume}, capacity=128, totalAddedValues=2}\n")),(0,n.yg)("h2",{id:"design-notes"},"Design Notes"),(0,n.yg)("p",null,"For the initial implementation no specific OpenTelemetry infrastructure is required, however existing environment variables\nfor OpenTelemetry can continue to be used and will export the new spans if configured."),(0,n.yg)("p",null,"The Trace API implementation does not rely on any additional external systems or infrastructure. Due to this design\nchoice, the trace is determined by inspecting the 3 storage systems (Primary Storage (SQL/Cassandra), Elasticsearch/Opensearch,\nKafka topics) for the ",(0,n.yg)("inlineCode",{parentName:"p"},"trace id")," or related timestamps."),(0,n.yg)("p",null,"The ",(0,n.yg)("inlineCode",{parentName:"p"},"trace id")," is stored in systemMetadata in both SQL and ES. For ES specifically, the presence of the ",(0,n.yg)("inlineCode",{parentName:"p"},"trace id")," in\nthe system metadata index is used as a proxy to determine a successful write to ES."),(0,n.yg)("p",null,"The tracing feature will additionally fetch messages from the kafka topics (including the failed MCP topic) for\nmore detailed error information. Pending states are derived from offsets of the message vs the current offsets of the\nconsumer groups."))}g.isMDXComponent=!0}}]);