yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-06-27 05:03:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(opentelemetry): Resolve type mismatch in metrics exporter (#13053)

Browse Source 
Co-authored-by: Pedro Silva <pedro@acryl.io>
This commit is contained in:
Rafael Sousa 2025-04-04 11:51:10 -03:00 committed by GitHub
parent b37fa03846
commit c1f6bd171e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 108 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build.gradle
View File

@ -226,8 +226,11 @@ project.ext.externalDependency = [
'opentelemetryApi': 'io.opentelemetry:opentelemetry-api:' + openTelemetryVersion,
'opentelemetrySdk': 'io.opentelemetry:opentelemetry-sdk:' + openTelemetryVersion,
'opentelemetrySdkTrace': 'io.opentelemetry:opentelemetry-sdk-trace:' + openTelemetryVersion,
'opentelemetrySdkMetrics': 'io.opentelemetry:opentelemetry-sdk-metrics:' + openTelemetryVersion,
'opentelemetryAutoConfig': 'io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:' + openTelemetryVersion,
'opentelemetryExporter': 'io.opentelemetry:opentelemetry-exporter-otlp:' + openTelemetryVersion,
'openTelemetryExporterLogging': 'io.opentelemetry:opentelemetry-exporter-logging:' + openTelemetryVersion,
'openTelemetryExporterCommon': 'io.opentelemetry:opentelemetry-exporter-otlp-common:' + openTelemetryVersion,
'opentelemetryAnnotations': 'io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations:2.11.0',
'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
'parquet': 'org.apache.parquet:parquet-avro:1.15.1',

2
datahub-frontend/play.gradle
View File

@ -95,6 +95,8 @@ dependencies {
compileOnly externalDependency.lombok
runtimeOnly externalDependency.guicePlay
runtimeOnly externalDependency.opentelemetryExporter
runtimeOnly externalDependency.openTelemetryExporterLogging
runtimeOnly externalDependency.openTelemetryExporterCommon
runtimeOnly (externalDependency.playDocs) {
exclude group: 'com.typesafe.akka', module: "akka-http-core_$playScalaVersion"
}

2
datahub-upgrade/build.gradle
View File

@ -72,6 +72,8 @@ dependencies {
implementation externalDependency.springCore
implementation externalDependency.springKafka
runtimeOnly externalDependency.opentelemetryExporter
runtimeOnly externalDependency.openTelemetryExporterLogging
runtimeOnly externalDependency.openTelemetryExporterCommon
runtimeOnly externalDependency.logbackClassic
runtimeOnly externalDependency.mariadbConnector

14
docker/datahub-actions/env/docker.env vendored
View File

@ -36,3 +36,17 @@ DATAHUB_ACTIONS_TEAMS_ENABLED
DATAHUB_ACTIONS_TEAMS_DATAHUB_BASE_URL
DATAHUB_ACTIONS_TEAMS_WEBHOOK_URL
DATAHUB_ACTIONS_TEAMS_SUPPRESS_SYSTEM_ACTIVITY
# Uncomment the following to enable monitoring
# ENABLE_PROMETHEUS=true
# DATAHUB_DEBUG=true
# ENABLE_OTEL=true
# OTEL_RESOURCE_ATTRIBUTE={"service.name": "datahub-actions"}
# OTEL_SERVICE_NAME=datahub-actions
# OTEL_LOG_LEVEL=debug
# OTEL_METRICS_EXPORTER=console
# OTEL_TRACES_EXPORTER=console
# OTEL_LOGS_EXPORTER=console
# OTEL_SDK_DISABLED=false
# OTEL_INSTRUMENTATION_COMMON_DEFAULT_ENABLED=true
# OTEL_PROPAGATORS=tracecontext,baggage

14
docker/datahub-frontend/env/docker.env vendored
View File

@ -69,3 +69,17 @@ ELASTIC_CLIENT_PORT=9200
# set ELASTIC_CLIENT_USE_SSL=true and uncomment:
# ELASTIC_CLIENT_USERNAME=
# ELASTIC_CLIENT_PASSWORD=
# Uncomment the following to enable monitoring
# ENABLE_PROMETHEUS=true
# DATAHUB_DEBUG=true
# ENABLE_OTEL=true
# OTEL_RESOURCE_ATTRIBUTE={"service.name": "datahub-frontend"}
# OTEL_SERVICE_NAME=datahub-frontend
# OTEL_LOG_LEVEL=debug
# OTEL_METRICS_EXPORTER=console
# OTEL_TRACES_EXPORTER=console
# OTEL_LOGS_EXPORTER=console
# OTEL_SDK_DISABLED=false
# OTEL_INSTRUMENTATION_COMMON_DEFAULT_ENABLED=true
# OTEL_PROPAGATORS=tracecontext,baggage

14
docker/datahub-gms/env/docker.env vendored
View File

@ -85,3 +85,17 @@ THEME_V2_DEFAULT=true
# Uncomment to run a one-time upgrade to migrate legacy default browse path format to latest format
# More details can be found at https://datahubproject.io/docs/advanced/browse-paths-upgrade
# UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED=true
# Uncomment the following to enable monitoring
# ENABLE_PROMETHEUS=true
# DATAHUB_DEBUG=true
# ENABLE_OTEL=true
# OTEL_RESOURCE_ATTRIBUTE={"service.name": "datahub-gms"}
# OTEL_SERVICE_NAME=datahub-gms
# OTEL_LOG_LEVEL=debug
# OTEL_METRICS_EXPORTER=console
# OTEL_TRACES_EXPORTER=console
# OTEL_LOGS_EXPORTER=console
# OTEL_SDK_DISABLED=false
# OTEL_INSTRUMENTATION_COMMON_DEFAULT_ENABLED=true
# OTEL_PROPAGATORS=tracecontext,baggage

14
docker/datahub-mae-consumer/env/docker.env vendored
View File

@ -39,3 +39,17 @@ ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-regis
# ELASTICSEARCH_SSL_KEYSTORE_FILE=
# ELASTICSEARCH_SSL_KEYSTORE_TYPE=
# ELASTICSEARCH_SSL_KEYSTORE_PASSWORD=
# Uncomment the following to enable monitoring
# ENABLE_PROMETHEUS=true
# DATAHUB_DEBUG=true
# ENABLE_OTEL=true
# OTEL_RESOURCE_ATTRIBUTE={"service.name": "datahub-mae-consumer"}
# OTEL_SERVICE_NAME=datahub-mae-consumer
# OTEL_LOG_LEVEL=debug
# OTEL_METRICS_EXPORTER=console
# OTEL_TRACES_EXPORTER=console
# OTEL_LOGS_EXPORTER=console
# OTEL_SDK_DISABLED=false
# OTEL_INSTRUMENTATION_COMMON_DEFAULT_ENABLED=true
# OTEL_PROPAGATORS=tracecontext,baggage

14
docker/datahub-mce-consumer/env/docker.env vendored
View File

@ -47,3 +47,17 @@ UI_INGESTION_ENABLED=false
# set ELASTICSEARCH_USE_SSL=true and uncomment:
# ELASTICSEARCH_USERNAME=
# ELASTICSEARCH_PASSWORD=
# Uncomment the following to enable monitoring
# ENABLE_PROMETHEUS=true
# DATAHUB_DEBUG=true
# ENABLE_OTEL=true
# OTEL_RESOURCE_ATTRIBUTE={"service.name": "datahub-mce-consumer"}
# OTEL_SERVICE_NAME=datahub-mce-consumer
# OTEL_LOG_LEVEL=debug
# OTEL_METRICS_EXPORTER=console
# OTEL_TRACES_EXPORTER=console
# OTEL_LOGS_EXPORTER=console
# OTEL_SDK_DISABLED=false
# OTEL_INSTRUMENTATION_COMMON_DEFAULT_ENABLED=true
# OTEL_PROPAGATORS=tracecontext,baggage

1
metadata-io/build.gradle
View File

@ -59,6 +59,7 @@ dependencies {
implementation externalDependency.ebeanAgent
implementation externalDependency.opentelemetryAnnotations
implementation externalDependency.opentelemetrySdkTrace
implementation externalDependency.opentelemetrySdkMetrics
implementation externalDependency.resilience4j
// Newer Spring libraries require JDK17 classes, allow for JDK11
compileOnly externalDependency.springBootAutoconfigureJdk11

2
metadata-jobs/mae-consumer-job/build.gradle
View File

@ -30,6 +30,8 @@ dependencies {
compileOnly externalDependency.lombok
implementation externalDependency.logbackClassic
runtimeOnly externalDependency.opentelemetryExporter
runtimeOnly externalDependency.openTelemetryExporterLogging
runtimeOnly externalDependency.openTelemetryExporterCommon
testImplementation project(':metadata-dao-impl:kafka-producer')
testImplementation project(':metadata-jobs:pe-consumer')

1
metadata-jobs/mae-consumer/build.gradle
View File

@ -34,6 +34,7 @@ dependencies {
implementation externalDependency.springKafka
implementation externalDependency.annotationApi
implementation externalDependency.opentelemetrySdkTrace
implementation externalDependency.opentelemetrySdkMetrics
implementation externalDependency.slf4jApi
compileOnly externalDependency.lombok

2
metadata-jobs/mce-consumer-job/build.gradle
View File

@ -32,6 +32,8 @@ dependencies {
compileOnly externalDependency.lombok
implementation externalDependency.logbackClassic
runtimeOnly externalDependency.opentelemetryExporter
runtimeOnly externalDependency.openTelemetryExporterLogging
runtimeOnly externalDependency.openTelemetryExporterCommon
runtimeOnly externalDependency.mariadbConnector
runtimeOnly externalDependency.mysqlConnector

1
metadata-jobs/mce-consumer/build.gradle
View File

@ -31,6 +31,7 @@ dependencies {
implementation externalDependency.springKafka
implementation externalDependency.opentelemetrySdkTrace
implementation externalDependency.opentelemetrySdkMetrics
implementation externalDependency.slf4jApi
compileOnly externalDependency.lombok

1
metadata-operation-context/build.gradle
View File

@ -15,6 +15,7 @@ dependencies {
implementation spec.product.pegasus.restliServer
implementation externalDependency.opentelemetryApi
implementation externalDependency.opentelemetrySdkTrace
implementation externalDependency.opentelemetrySdkMetrics
compileOnly externalDependency.lombok
annotationProcessor externalDependency.lombok

1
metadata-service/factories/build.gradle
View File

@ -54,6 +54,7 @@ dependencies {
implementation externalDependency.opentelemetryApi
implementation externalDependency.opentelemetrySdk
implementation externalDependency.opentelemetrySdkTrace
implementation externalDependency.opentelemetrySdkMetrics
implementation externalDependency.opentelemetryAutoConfig
testImplementation externalDependency.springBootTest

23
metadata-service/factories/src/main/java/com/linkedin/gms/factory/system_telemetry/OpenTelemetryBaseFactory.java
View File

@ -8,6 +8,7 @@ import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
import io.opentelemetry.sdk.metrics.export.MetricExporter;
import io.opentelemetry.sdk.resources.Resource;
import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
@ -51,14 +52,26 @@ public abstract class OpenTelemetryBaseFactory {
.addTracerProviderCustomizer(
(sdkTracerProviderBuilder, configProperties) ->
sdkTracerProviderBuilder
.addSpanProcessor(SimpleSpanProcessor.create(TraceContext.LOG_SPAN_EXPORTER))
.addSpanProcessor(
TraceContext.LOG_SPAN_EXPORTER != null
? SimpleSpanProcessor.create(TraceContext.LOG_SPAN_EXPORTER)
: SimpleSpanProcessor.create(
new MetricSpanExporter())) // Fallback for exporter
.addSpanProcessor(BatchSpanProcessor.builder(new MetricSpanExporter()).build())
.setIdGenerator(TraceContext.TRACE_ID_GENERATOR)
.setIdGenerator(
TraceContext.TRACE_ID_GENERATOR != null
? TraceContext.TRACE_ID_GENERATOR
: io.opentelemetry.sdk.trace.IdGenerator
.random()) // Fallback for ID generator
.setResource(
Resource.getDefault()
.merge(
Resource.create(
Attributes.of(SERVICE_NAME, getApplicationComponent())))))
Attributes.of(
SERVICE_NAME,
getApplicationComponent() != null
? getApplicationComponent()
: "default-service")))))
.addPropagatorCustomizer(
(existingPropagator, configProperties) -> {
// If OTEL_PROPAGATORS is not set or doesn't include tracecontext,
@ -72,7 +85,9 @@ public abstract class OpenTelemetryBaseFactory {
(metricExporter, configProperties) -> {
String metricsExporter = configProperties.getString("OTEL_METRICS_EXPORTER");
return (metricsExporter == null || metricsExporter.trim().isEmpty())
? null // Return null to disable the exporter
? (metricExporter != null
? metricExporter
: (MetricExporter) new MetricSpanExporter())
: metricExporter;
})
.build()

1
metadata-service/openapi-servlet/build.gradle
View File

@ -39,6 +39,7 @@ dependencies {
implementation externalDependency.opentelemetryApi
implementation externalDependency.opentelemetrySdk
implementation externalDependency.opentelemetrySdkTrace
implementation externalDependency.opentelemetrySdkMetrics
testImplementation externalDependency.springBootTest
testImplementation project(':mock-entity-registry')

2
metadata-service/war/build.gradle
View File

@ -44,6 +44,8 @@ dependencies {
implementation externalDependency.springBootAutoconfigure
implementation externalDependency.servletApi
runtimeOnly externalDependency.opentelemetryExporter
runtimeOnly externalDependency.openTelemetryExporterLogging
runtimeOnly externalDependency.openTelemetryExporterCommon
implementation spec.product.pegasus.restliDocgen
implementation spec.product.pegasus.restliSpringBridge