diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index abc17a31e3d..af1a0fc4b9e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -56,6 +56,7 @@ import org.openmetadata.schema.TokenInterface; import org.openmetadata.schema.analytics.ReportData; import org.openmetadata.schema.analytics.WebAnalyticEvent; import org.openmetadata.schema.api.configuration.LoginConfiguration; +import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration; import org.openmetadata.schema.auth.EmailVerificationToken; import org.openmetadata.schema.auth.PasswordResetToken; import org.openmetadata.schema.auth.PersonalAccessToken; @@ -3939,6 +3940,7 @@ public interface CollectionDAO { case SLACK_APP_CONFIGURATION -> JsonUtils.readValue(json, String.class); case SLACK_BOT, SLACK_INSTALLER -> JsonUtils.readValue( json, new TypeReference>() {}); + case PROFILER_CONFIGURATION -> JsonUtils.readValue(json, ProfilerConfiguration.class); default -> throw new IllegalArgumentException("Invalid Settings Type " + configType); }; settings.setConfigValue(value); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java index e6217dfc920..855dc3c0a06 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java @@ -10,6 +10,8 @@ import io.dropwizard.jackson.Jackson; import io.dropwizard.jersey.validation.Validators; import java.io.IOException; import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; import javax.validation.Validator; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; @@ -23,6 +25,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.TestMethodOrder; import org.openmetadata.api.configuration.LogoConfiguration; +import org.openmetadata.schema.api.configuration.profiler.MetricConfigurationDefinition; +import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration; import org.openmetadata.schema.api.data.*; import org.openmetadata.schema.api.services.CreateDashboardService; import org.openmetadata.schema.api.services.CreateDatabaseService; @@ -37,10 +41,12 @@ import org.openmetadata.schema.auth.SSOAuthMechanism; import org.openmetadata.schema.email.SmtpSettings; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.teams.AuthenticationMechanism; +import org.openmetadata.schema.profiler.MetricType; import org.openmetadata.schema.security.client.GoogleSSOClientConfig; import org.openmetadata.schema.settings.Settings; import org.openmetadata.schema.settings.SettingsType; import org.openmetadata.schema.system.ValidationResponse; +import org.openmetadata.schema.type.ColumnDataType; import org.openmetadata.schema.util.EntitiesCount; import org.openmetadata.schema.util.ServicesCount; import org.openmetadata.service.OpenMetadataApplicationConfig; @@ -310,6 +316,57 @@ public class SystemResourceTest extends OpenMetadataApplicationTest { Assertions.assertEquals(Boolean.TRUE, response.getMigrations().getPassed()); } + @Test + void globalProfilerConfig(TestInfo test) throws HttpResponseException { + // Create a profiler config + ProfilerConfiguration profilerConfiguration = new ProfilerConfiguration(); + MetricConfigurationDefinition intMetricConfigDefinition = + new MetricConfigurationDefinition() + .withDataType(ColumnDataType.INT) + .withMetrics(List.of(MetricType.COUNT, MetricType.FIRST_QUARTILE, MetricType.MEAN)); + MetricConfigurationDefinition dateTimeMetricConfigDefinition = + new MetricConfigurationDefinition() + .withDataType(ColumnDataType.DATETIME) + .withDisabled(true); + profilerConfiguration.setMetricConfiguration( + List.of(intMetricConfigDefinition, dateTimeMetricConfigDefinition)); + Settings profilerSettings = + new Settings() + .withConfigType(SettingsType.PROFILER_CONFIGURATION) + .withConfigValue(profilerConfiguration); + createSystemConfig(profilerSettings); + ProfilerConfiguration createdProfilerSettings = + JsonUtils.convertValue( + getSystemConfig(SettingsType.PROFILER_CONFIGURATION).getConfigValue(), + ProfilerConfiguration.class); + Assertions.assertEquals(profilerConfiguration, createdProfilerSettings); + + // Update the profiler config + profilerConfiguration.setMetricConfiguration(List.of(intMetricConfigDefinition)); + profilerSettings = + new Settings() + .withConfigType(SettingsType.PROFILER_CONFIGURATION) + .withConfigValue(profilerConfiguration); + updateSystemConfig(profilerSettings); + ProfilerConfiguration updatedProfilerSettings = + JsonUtils.convertValue( + getSystemConfig(SettingsType.PROFILER_CONFIGURATION).getConfigValue(), + ProfilerConfiguration.class); + Assertions.assertEquals(profilerConfiguration, updatedProfilerSettings); + + // Delete the profiler config + profilerConfiguration.setMetricConfiguration(new ArrayList<>()); + updateSystemConfig( + new Settings() + .withConfigType(SettingsType.PROFILER_CONFIGURATION) + .withConfigValue(profilerConfiguration)); + updatedProfilerSettings = + JsonUtils.convertValue( + getSystemConfig(SettingsType.PROFILER_CONFIGURATION).getConfigValue(), + ProfilerConfiguration.class); + Assertions.assertEquals(profilerConfiguration, updatedProfilerSettings); + } + private static ValidationResponse getValidation() throws HttpResponseException { WebTarget target = getResource("system/status"); return TestUtils.get(target, ValidationResponse.class, ADMIN_AUTH_HEADERS); @@ -334,4 +391,9 @@ public class SystemResourceTest extends OpenMetadataApplicationTest { WebTarget target = getResource("system/settings"); TestUtils.put(target, updatedSetting, Response.Status.OK, ADMIN_AUTH_HEADERS); } + + private static void createSystemConfig(Settings updatedSetting) throws HttpResponseException { + WebTarget target = getResource("system/settings"); + TestUtils.put(target, updatedSetting, Response.Status.CREATED, ADMIN_AUTH_HEADERS); + } } diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/profilerConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/profilerConfiguration.json new file mode 100644 index 00000000000..a8dbcf4defb --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/profilerConfiguration.json @@ -0,0 +1,81 @@ +{ + "$id": "https://open-metadata.org/schema/settings/profilerConfiguration.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ProfilerConfiguration", + "description": "This schema defines the profiler configuration. It is used to configure globally the metrics to compute for specific data types.", + "type": "object", + "javaType": "org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration", + "definitions": { + "metricType": { + "javaType": "org.openmetadata.schema.profiler.MetricType", + "description": "This schema defines all possible metric types in OpenMetadata.", + "type": "string", + "enum": [ + "MEAN", + "COUNT", + "COUNT_IN_SET", + "COLUMN_COUNT", + "DISTINCT_COUNT", + "DISTINCT_RATIO", + "ILIKE_COUNT", + "LIKE_COUNT", + "NOT_LIKE_COUNT", + "REGEX_COUNT", + "NOT_REGEX_COUNT", + "MAX", + "MAX_LENGTH", + "MIN", + "MIN_LENGTH", + "NULL_COUNT", + "ROW_COUNT", + "STDDEV", + "SUM", + "UNIQUE_COUNT", + "UNIQUE_RATIO", + "COLUMN_NAMES", + "DUPLICATE_COUNT", + "ILIKE_RATIO", + "LIKE_RATIO", + "NULL_RATIO", + "IQR", + "NON_PARAMETRIC_SKEW", + "MEDIAN", + "FIRST_QUARTILE", + "THIRD_QUARTILE", + "SYSTEM", + "HISTOGRAM" + ] + }, + "metricConfigurationDefinition": { + "type": "object", + "javaType": "org.openmetadata.schema.api.configuration.profiler.MetricConfigurationDefinition", + "description": "This schema defines the parameters that can be passed for a Test Case.", + "properties": { + "dataType": { + "$ref": "../entity/data/table.json#/definitions/dataType" + }, + "metrics": { + "type": "array", + "items": { + "$ref": "#/definitions/metricType" + } + }, + "disabled": { + "type": "boolean", + "description": "If true, the metric will not be computed for the data type.", + "default": false + } + }, + "additionalProperties": false + } + }, + "properties": { + "metricConfiguration": { + "type": "array", + "items": { + "$ref": "#/definitions/metricConfigurationDefinition" + } + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/settings/settings.json b/openmetadata-spec/src/main/resources/json/schema/settings/settings.json index 71506695541..6fb7816e372 100644 --- a/openmetadata-spec/src/main/resources/json/schema/settings/settings.json +++ b/openmetadata-spec/src/main/resources/json/schema/settings/settings.json @@ -27,7 +27,8 @@ "loginConfiguration", "slackAppConfiguration", "slackBot", - "slackInstaller" + "slackInstaller", + "profilerConfiguration" ] } }, @@ -67,8 +68,10 @@ }, { "$ref": "../configuration/slackAppConfiguration.json" + }, + { + "$ref": "../configuration/profilerConfiguration.json" } - ] } },