Fix #1607 Operational metrics for OpenMetadata system observability (#4386)

Co-authored-by: Akash-Jain <Akash.J@deuexsolutions.com>

catalog-rest-service/pom.xml

@@ -94,6 +94,14 @@
       <groupId>org.glassfish.jersey.media</groupId>
       <artifactId>jersey-media-multipart</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.github.maksymdolgykh.dropwizard</groupId>
+      <artifactId>dropwizard-micrometer-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.github.maksymdolgykh.dropwizard</groupId>
+      <artifactId>dropwizard-micrometer-jdbi</artifactId>
+    </dependency>
     <dependency>
       <groupId>commons-lang</groupId>
       <artifactId>commons-lang</artifactId>

catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java

@@ -29,10 +29,16 @@ import io.dropwizard.setup.Bootstrap;
 import io.dropwizard.setup.Environment;
 import io.federecio.dropwizard.swagger.SwaggerBundle;
 import io.federecio.dropwizard.swagger.SwaggerBundleConfiguration;
+import io.github.maksymdolgykh.dropwizard.micrometer.MicrometerBundle;
+import io.github.maksymdolgykh.dropwizard.micrometer.MicrometerHttpFilter;
+import io.github.maksymdolgykh.dropwizard.micrometer.MicrometerJdbiTimingCollector;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.time.temporal.ChronoUnit;
+import java.util.EnumSet;
 import java.util.Optional;
+import javax.servlet.DispatcherType;
+import javax.servlet.FilterRegistration;
 import javax.ws.rs.container.ContainerRequestFilter;
 import javax.ws.rs.container.ContainerResponseFilter;
 import javax.ws.rs.core.Response;
@@ -79,6 +85,7 @@ public class CatalogApplication extends Application<CatalogApplicationConfig> {
       throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException,
           InvocationTargetException, IOException {
     final Jdbi jdbi = new JdbiFactory().build(environment, catalogConfig.getDataSourceFactory(), "database");
+    jdbi.setTimingCollector(new MicrometerJdbiTimingCollector());
 
     SqlLogger sqlLogger =
         new SqlLogger() {
@@ -125,7 +132,7 @@ public class CatalogApplication extends Application<CatalogApplicationConfig> {
     environment.jersey().register(new JsonProcessingExceptionMapper(true));
     environment.jersey().register(new EarlyEofExceptionMapper());
     environment.jersey().register(JsonMappingExceptionMapper.class);
-    environment.healthChecks().register("UserDatabaseCheck", new CatalogHealthCheck(jdbi));
+    environment.healthChecks().register("OpenMetadataServerHealthCheck", new OpenMetadataServerHealthCheck());
     registerResources(catalogConfig, environment, jdbi);
     RoleEvaluator.getInstance().load();
     PolicyEvaluator.getInstance().load();
@@ -137,9 +144,13 @@ public class CatalogApplication extends Application<CatalogApplicationConfig> {
     EventPubSub.start();
     // Register Event publishers
     registerEventPublisher(catalogConfig);
+
     // start authorizer after event publishers
     // authorizer creates admin/bot users, ES publisher should start before to index users created by authorizer
     authorizer.init(catalogConfig.getAuthorizerConfiguration(), jdbi);
+    FilterRegistration.Dynamic micrometerFilter =
+        environment.servlets().addFilter("MicrometerHttpFilter", new MicrometerHttpFilter());
+    micrometerFilter.addMappingForUrlPatterns(EnumSet.allOf(DispatcherType.class), true, "/*");
   }
 
   @SneakyThrows
@@ -163,6 +174,7 @@ public class CatalogApplication extends Application<CatalogApplicationConfig> {
             return configuration.getHealthConfiguration();
           }
         });
+    bootstrap.addBundle(new MicrometerBundle());
     super.initialize(bootstrap);
   }
 

catalog-rest-service/src/main/java/org/openmetadata/catalog/OpenMetadataServerHealthCheck.java

@@ -0,0 +1,26 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.openmetadata.catalog;
+
+import com.codahale.metrics.health.HealthCheck;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class OpenMetadataServerHealthCheck extends HealthCheck {
+
+  @Override
+  protected Result check() throws Exception {
+    return Result.healthy();
+  }
+}

catalog-rest-service/src/test/resources/openmetadata-secure-test.yaml

@@ -114,9 +114,8 @@ migrationConfiguration:
 health:
   delayedShutdownHandlerEnabled: true
   shutdownWaitPeriod: 5s
-  healthCheckUrlPaths: ["/api/v1/health-check"]
   healthChecks:
-    - name: UserDatabaseCheck
+    - name: OpenMetadataServerHealthCheck
       critical: true
 
 # Authorizer Configuration

conf/openmetadata.yaml

@@ -221,9 +221,8 @@ fernetConfiguration:
 health:
   delayedShutdownHandlerEnabled: true
   shutdownWaitPeriod: 1s
-  healthCheckUrlPaths: ["/api/v1/health-check"]
   healthChecks:
-    - name: UserDatabaseCheck
+    - name: OpenMetadataServerHealthCheck
       critical: true
       schedule:
         checkInterval: 2500ms

docker/local-metadata/docker-compose.yml

@@ -98,11 +98,13 @@ services:
       OM_AUTH_AIRFLOW_CUSTOM_OIDC_TOKEN_ENDPOINT_URL: ${OM_AUTH_AIRFLOW_CUSTOM_OIDC_TOKEN_ENDPOINT_URL:-""}
     expose:
       - 8585
+      - 8586
       - 9200
       - 9300
       - 3306
     ports:
       - 8585:8585
+      - 8586:8586
     depends_on:
       - elasticsearch
       - mysql

docker/metadata/docker-compose.yml

@@ -87,11 +87,13 @@ services:
       OM_AUTH_AIRFLOW_CUSTOM_OIDC_TOKEN_ENDPOINT_URL: ${OM_AUTH_AIRFLOW_CUSTOM_OIDC_TOKEN_ENDPOINT_URL:-""}
     expose:
       - 8585
+      - 8586
       - 9200
       - 9300
       - 3306
     ports:
       - 8585:8585
+      - 8586:8586
     depends_on:
       - mysql
     networks:

ingestion/src/metadata/ingestion/ometa/ometa_api.py

@@ -649,9 +649,10 @@ class OpenMetadata(
 
     def health_check(self) -> bool:
         """
-        Run endpoint health-check. Return `true` if OK
+        Run version api call. Return `true` if response is not None
         """
-        return self.client.get("/health-check")["status"] == "healthy"
+        raw_version = self.client.get("/version")["version"]
+        return raw_version is not None
 
     def close(self):
         """

ingestion/tests/integration/source/hive/test_hive_crud.py

@@ -154,7 +154,7 @@ def test_read_tables(hive_service):
 
 
 def test_check_table():
-    is_responsive("http://localhost:8585/api/v1/health-check")
+    is_responsive("http://localhost:8586/healthcheck")
     metadata_config = MetadataServerConfig.parse_obj(
         {"api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth"}
     )

ingestion/tests/integration/source/trino/test_trino_crud.py

@@ -162,7 +162,7 @@ def test_read_tables(trino_service):
 
 
 def test_check_table():
-    is_responsive("http://localhost:8585/api/v1/health-check")
+    is_responsive("http://localhost:8586/healthcheck")
     metadata_config = MetadataServerConfig.parse_obj(
         {"api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth"}
     )

pom.xml

@@ -95,6 +95,7 @@
     <sonar.skip>false</sonar.skip>
     <sonar.dynamicAnalysis>reuseReports</sonar.dynamicAnalysis>
     <testng.version>7.6.0</testng.version>
+    <dropwizard-micrometer.version>2.0.5</dropwizard-micrometer.version>
   </properties>
   <dependencyManagement>
     <dependencies>
@@ -318,6 +319,16 @@
         <artifactId>spring-core</artifactId>
         <version>${spring.version}</version>
       </dependency>
+      <dependency>
+        <groupId>io.github.maksymdolgykh.dropwizard</groupId>
+        <artifactId>dropwizard-micrometer-core</artifactId>
+        <version>${dropwizard-micrometer.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.github.maksymdolgykh.dropwizard</groupId>
+        <artifactId>dropwizard-micrometer-jdbi</artifactId>
+        <version>${dropwizard-micrometer.version}</version>
+      </dependency>
       <!-- API, java.xml.bind module -->
       <dependency>
         <groupId>jakarta.xml.bind</groupId>