Fix #11951 [backend] - Implement Profiler Global Config (#15889)

* feat: add global metric configuration for the profiler * style: ran java linting * fix: renamed disable to disabled * style: ran java linting * fix: json schema profiler config title
2025-10-31 10:39:30 +00:00 · 2024-04-16 18:56:21 +02:00 · 2024-04-16 18:56:21 +02:00 · 8d377f5c99
commit 8d377f5c99
parent c7e03471a1
4 changed files with 150 additions and 2 deletions
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
@ -56,6 +56,7 @@ import org.openmetadata.schema.TokenInterface;
 import org.openmetadata.schema.analytics.ReportData;
 import org.openmetadata.schema.analytics.WebAnalyticEvent;
 import org.openmetadata.schema.api.configuration.LoginConfiguration;
+import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration;
 import org.openmetadata.schema.auth.EmailVerificationToken;
 import org.openmetadata.schema.auth.PasswordResetToken;
 import org.openmetadata.schema.auth.PersonalAccessToken;
@ -3939,6 +3940,7 @@ public interface CollectionDAO {
            case SLACK_APP_CONFIGURATION -> JsonUtils.readValue(json, String.class);
            case SLACK_BOT, SLACK_INSTALLER -> JsonUtils.readValue(
                json, new TypeReference<HashMap<String, Object>>() {});
+            case PROFILER_CONFIGURATION -> JsonUtils.readValue(json, ProfilerConfiguration.class);
            default -> throw new IllegalArgumentException("Invalid Settings Type " + configType);
          };
      settings.setConfigValue(value);
--- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java
@ -10,6 +10,8 @@ import io.dropwizard.jackson.Jackson;
 import io.dropwizard.jersey.validation.Validators;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
 import javax.validation.Validator;
 import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.Response;
@ -23,6 +25,8 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.TestMethodOrder;
 import org.openmetadata.api.configuration.LogoConfiguration;
+import org.openmetadata.schema.api.configuration.profiler.MetricConfigurationDefinition;
+import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration;
 import org.openmetadata.schema.api.data.*;
 import org.openmetadata.schema.api.services.CreateDashboardService;
 import org.openmetadata.schema.api.services.CreateDatabaseService;
@ -37,10 +41,12 @@ import org.openmetadata.schema.auth.SSOAuthMechanism;
 import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.entity.data.Table;
 import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
+import org.openmetadata.schema.profiler.MetricType;
 import org.openmetadata.schema.security.client.GoogleSSOClientConfig;
 import org.openmetadata.schema.settings.Settings;
 import org.openmetadata.schema.settings.SettingsType;
 import org.openmetadata.schema.system.ValidationResponse;
+import org.openmetadata.schema.type.ColumnDataType;
 import org.openmetadata.schema.util.EntitiesCount;
 import org.openmetadata.schema.util.ServicesCount;
 import org.openmetadata.service.OpenMetadataApplicationConfig;
@ -310,6 +316,57 @@ public class SystemResourceTest extends OpenMetadataApplicationTest {
    Assertions.assertEquals(Boolean.TRUE, response.getMigrations().getPassed());
  }

+  @Test
+  void globalProfilerConfig(TestInfo test) throws HttpResponseException {
+    // Create a profiler config
+    ProfilerConfiguration profilerConfiguration = new ProfilerConfiguration();
+    MetricConfigurationDefinition intMetricConfigDefinition =
+        new MetricConfigurationDefinition()
+            .withDataType(ColumnDataType.INT)
+            .withMetrics(List.of(MetricType.COUNT, MetricType.FIRST_QUARTILE, MetricType.MEAN));
+    MetricConfigurationDefinition dateTimeMetricConfigDefinition =
+        new MetricConfigurationDefinition()
+            .withDataType(ColumnDataType.DATETIME)
+            .withDisabled(true);
+    profilerConfiguration.setMetricConfiguration(
+        List.of(intMetricConfigDefinition, dateTimeMetricConfigDefinition));
+    Settings profilerSettings =
+        new Settings()
+            .withConfigType(SettingsType.PROFILER_CONFIGURATION)
+            .withConfigValue(profilerConfiguration);
+    createSystemConfig(profilerSettings);
+    ProfilerConfiguration createdProfilerSettings =
+        JsonUtils.convertValue(
+            getSystemConfig(SettingsType.PROFILER_CONFIGURATION).getConfigValue(),
+            ProfilerConfiguration.class);
+    Assertions.assertEquals(profilerConfiguration, createdProfilerSettings);
+
+    // Update the profiler config
+    profilerConfiguration.setMetricConfiguration(List.of(intMetricConfigDefinition));
+    profilerSettings =
+        new Settings()
+            .withConfigType(SettingsType.PROFILER_CONFIGURATION)
+            .withConfigValue(profilerConfiguration);
+    updateSystemConfig(profilerSettings);
+    ProfilerConfiguration updatedProfilerSettings =
+        JsonUtils.convertValue(
+            getSystemConfig(SettingsType.PROFILER_CONFIGURATION).getConfigValue(),
+            ProfilerConfiguration.class);
+    Assertions.assertEquals(profilerConfiguration, updatedProfilerSettings);
+
+    // Delete the profiler config
+    profilerConfiguration.setMetricConfiguration(new ArrayList<>());
+    updateSystemConfig(
+        new Settings()
+            .withConfigType(SettingsType.PROFILER_CONFIGURATION)
+            .withConfigValue(profilerConfiguration));
+    updatedProfilerSettings =
+        JsonUtils.convertValue(
+            getSystemConfig(SettingsType.PROFILER_CONFIGURATION).getConfigValue(),
+            ProfilerConfiguration.class);
+    Assertions.assertEquals(profilerConfiguration, updatedProfilerSettings);
+  }
+
  private static ValidationResponse getValidation() throws HttpResponseException {
    WebTarget target = getResource("system/status");
    return TestUtils.get(target, ValidationResponse.class, ADMIN_AUTH_HEADERS);
@ -334,4 +391,9 @@ public class SystemResourceTest extends OpenMetadataApplicationTest {
    WebTarget target = getResource("system/settings");
    TestUtils.put(target, updatedSetting, Response.Status.OK, ADMIN_AUTH_HEADERS);
  }
+
+  private static void createSystemConfig(Settings updatedSetting) throws HttpResponseException {
+    WebTarget target = getResource("system/settings");
+    TestUtils.put(target, updatedSetting, Response.Status.CREATED, ADMIN_AUTH_HEADERS);
+  }
 }
--- a/openmetadata-spec/src/main/resources/json/schema/configuration/profilerConfiguration.json
+++ b/openmetadata-spec/src/main/resources/json/schema/configuration/profilerConfiguration.json
@ -0,0 +1,81 @@
+{
+  "$id": "https://open-metadata.org/schema/settings/profilerConfiguration.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "ProfilerConfiguration",
+  "description": "This schema defines the profiler configuration. It is used to configure globally the metrics to compute for specific data types.",
+  "type": "object",
+  "javaType": "org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration",
+  "definitions": {
+    "metricType": {
+      "javaType": "org.openmetadata.schema.profiler.MetricType",
+      "description": "This schema defines all possible metric types in OpenMetadata.",
+      "type": "string",
+      "enum": [
+        "MEAN",
+        "COUNT",
+        "COUNT_IN_SET",
+        "COLUMN_COUNT",
+        "DISTINCT_COUNT",
+        "DISTINCT_RATIO",
+        "ILIKE_COUNT",
+        "LIKE_COUNT",
+        "NOT_LIKE_COUNT",
+        "REGEX_COUNT",
+        "NOT_REGEX_COUNT",
+        "MAX",
+        "MAX_LENGTH",
+        "MIN",
+        "MIN_LENGTH",
+        "NULL_COUNT",
+        "ROW_COUNT",
+        "STDDEV",
+        "SUM",
+        "UNIQUE_COUNT",
+        "UNIQUE_RATIO",
+        "COLUMN_NAMES",
+        "DUPLICATE_COUNT",
+        "ILIKE_RATIO",
+        "LIKE_RATIO",
+        "NULL_RATIO",
+        "IQR",
+        "NON_PARAMETRIC_SKEW",
+        "MEDIAN",
+        "FIRST_QUARTILE",
+        "THIRD_QUARTILE",
+        "SYSTEM",
+        "HISTOGRAM"
+      ]
+    },
+    "metricConfigurationDefinition": {
+      "type": "object",
+      "javaType": "org.openmetadata.schema.api.configuration.profiler.MetricConfigurationDefinition",
+      "description": "This schema defines the parameters that can be passed for a Test Case.",
+      "properties": {
+        "dataType": {
+          "$ref": "../entity/data/table.json#/definitions/dataType"
+        },
+        "metrics": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/metricType"
+          }
+        },
+        "disabled": {
+          "type": "boolean",
+          "description": "If true, the metric will not be computed for the data type.",
+          "default": false
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "properties": {
+      "metricConfiguration": {
+      "type": "array",
+      "items": {
+          "$ref": "#/definitions/metricConfigurationDefinition"
+      }
+    }
+  },
+  "additionalProperties": false
+}
--- a/openmetadata-spec/src/main/resources/json/schema/settings/settings.json
+++ b/openmetadata-spec/src/main/resources/json/schema/settings/settings.json
@ -27,7 +27,8 @@
        "loginConfiguration",
        "slackAppConfiguration",
        "slackBot",
-        "slackInstaller"
+        "slackInstaller",
+        "profilerConfiguration"
      ]
    }
  },
@ -67,8 +68,10 @@
        },
        {
          "$ref": "../configuration/slackAppConfiguration.json"
+        },
+        {
+          "$ref": "../configuration/profilerConfiguration.json"
        }
-        
      ]
    }
  },