From a68d16c7cacac0ac02611e04e4845c714e428f2d Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Mon, 4 Nov 2024 22:20:43 +0530 Subject: [PATCH] Lineage Settings --- .../service/jdbi3/CollectionDAO.java | 2 + .../resources/settings/SettingsCache.java | 17 ++ .../resources/system/SystemResourceTest.java | 147 ++++++++++++++++++ .../configuration/limitsConfiguration.json | 2 +- .../schema/configuration/lineageSettings.json | 43 +++++ .../schema/configuration/searchSettings.json | 16 ++ .../json/schema/settings/settings.json | 6 +- 7 files changed, 231 insertions(+), 2 deletions(-) create mode 100644 openmetadata-spec/src/main/resources/json/schema/configuration/lineageSettings.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/configuration/searchSettings.json diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 70f3418ba50..8bab5f64f8a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -58,6 +58,7 @@ import org.openmetadata.schema.analytics.ReportData; import org.openmetadata.schema.analytics.WebAnalyticEvent; import org.openmetadata.schema.api.configuration.LoginConfiguration; import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration; +import org.openmetadata.schema.api.lineage.LineageSettings; import org.openmetadata.schema.auth.EmailVerificationToken; import org.openmetadata.schema.auth.PasswordResetToken; import org.openmetadata.schema.auth.PersonalAccessToken; @@ -4629,6 +4630,7 @@ public interface CollectionDAO { case SLACK_APP_CONFIGURATION, SLACK_INSTALLER, SLACK_BOT, SLACK_STATE -> JsonUtils .readValue(json, String.class); case PROFILER_CONFIGURATION -> JsonUtils.readValue(json, ProfilerConfiguration.class); + case LINEAGE_SETTINGS -> JsonUtils.readValue(json, LineageSettings.class); default -> throw new IllegalArgumentException("Invalid Settings Type " + configType); }; settings.setConfigValue(value); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java index a3379da2753..3c0b6f7a89f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java @@ -15,6 +15,7 @@ package org.openmetadata.service.resources.settings; import static org.openmetadata.schema.settings.SettingsType.CUSTOM_UI_THEME_PREFERENCE; import static org.openmetadata.schema.settings.SettingsType.EMAIL_CONFIGURATION; +import static org.openmetadata.schema.settings.SettingsType.LINEAGE_SETTINGS; import static org.openmetadata.schema.settings.SettingsType.LOGIN_CONFIGURATION; import com.google.common.cache.CacheBuilder; @@ -28,6 +29,8 @@ import org.openmetadata.api.configuration.LogoConfiguration; import org.openmetadata.api.configuration.ThemeConfiguration; import org.openmetadata.api.configuration.UiThemePreference; import org.openmetadata.schema.api.configuration.LoginConfiguration; +import org.openmetadata.schema.api.lineage.LineageLayer; +import org.openmetadata.schema.api.lineage.LineageSettings; import org.openmetadata.schema.email.SmtpSettings; import org.openmetadata.schema.settings.Settings; import org.openmetadata.schema.settings.SettingsType; @@ -111,6 +114,20 @@ public class SettingsCache { .withJwtTokenExpiryTime(3600)); systemRepository.createNewSetting(setting); } + + Settings lineageSettings = systemRepository.getConfigWithKey(LINEAGE_SETTINGS.toString()); + if (lineageSettings == null) { + // Only in case a config doesn't exist in DB we insert it + Settings setting = + new Settings() + .withConfigType(LINEAGE_SETTINGS) + .withConfigValue( + new LineageSettings() + .withDownstreamDepth(2) + .withUpstreamDepth(2) + .withLineageLayer(LineageLayer.ENTITY_LINEAGE)); + systemRepository.createNewSetting(setting); + } } public static T getSetting(SettingsType settingName, Class clazz) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java index 97e781c09a5..083e9f49c5c 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java @@ -28,9 +28,11 @@ import org.junit.jupiter.api.TestMethodOrder; import org.openmetadata.api.configuration.LogoConfiguration; import org.openmetadata.api.configuration.ThemeConfiguration; import org.openmetadata.api.configuration.UiThemePreference; +import org.openmetadata.schema.api.configuration.LoginConfiguration; import org.openmetadata.schema.api.configuration.profiler.MetricConfigurationDefinition; import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration; import org.openmetadata.schema.api.data.*; +import org.openmetadata.schema.api.lineage.LineageSettings; import org.openmetadata.schema.api.services.CreateDashboardService; import org.openmetadata.schema.api.services.CreateDatabaseService; import org.openmetadata.schema.api.services.CreateMessagingService; @@ -70,6 +72,7 @@ import org.openmetadata.service.resources.services.MessagingServiceResourceTest; import org.openmetadata.service.resources.services.MlModelServiceResourceTest; import org.openmetadata.service.resources.services.PipelineServiceResourceTest; import org.openmetadata.service.resources.services.StorageServiceResourceTest; +import org.openmetadata.service.resources.settings.SettingsCache; import org.openmetadata.service.resources.storages.ContainerResourceTest; import org.openmetadata.service.resources.teams.TeamResourceTest; import org.openmetadata.service.resources.teams.UserResourceTest; @@ -344,6 +347,150 @@ public class SystemResourceTest extends OpenMetadataApplicationTest { assertEquals(Boolean.TRUE, response.getMigrations().getPassed()); } + @Test + void testDefaultSettingsInitialization() throws HttpResponseException { + SettingsCache.initialize(config); + Settings emailSettings = getSystemConfig(SettingsType.EMAIL_CONFIGURATION); + Settings uiThemeSettings = getSystemConfig(SettingsType.CUSTOM_UI_THEME_PREFERENCE); + SmtpSettings smtpSettings = + JsonUtils.convertValue(emailSettings.getConfigValue(), SmtpSettings.class); + assertEquals(config.getSmtpSettings().getUsername(), smtpSettings.getUsername()); + assertEquals(config.getSmtpSettings().getEmailingEntity(), smtpSettings.getEmailingEntity()); + UiThemePreference uiThemePreference = + JsonUtils.convertValue(uiThemeSettings.getConfigValue(), UiThemePreference.class); + assertEquals("", uiThemePreference.getCustomTheme().getPrimaryColor()); + assertEquals("", uiThemePreference.getCustomLogoConfig().getCustomLogoUrlPath()); + } + + @Test + void testEmailConfigurationSettings() throws HttpResponseException { + Settings emailSettings = getSystemConfig(SettingsType.EMAIL_CONFIGURATION); + SmtpSettings smtpSettings = + JsonUtils.convertValue(emailSettings.getConfigValue(), SmtpSettings.class); + SmtpSettings expectedSmtpSettings = config.getSmtpSettings(); + expectedSmtpSettings.setPassword( + smtpSettings.getPassword()); // Password is encrypted, so we use the stored one + assertEquals(expectedSmtpSettings, smtpSettings); + smtpSettings.setUsername("updatedUsername"); + smtpSettings.setEmailingEntity("updatedEntity"); + + Settings updatedEmailSettings = + new Settings() + .withConfigType(SettingsType.EMAIL_CONFIGURATION) + .withConfigValue(smtpSettings); + + updateSystemConfig(updatedEmailSettings); + + Settings updatedSettings = getSystemConfig(SettingsType.EMAIL_CONFIGURATION); + SmtpSettings updatedSmtpSettings = + JsonUtils.convertValue(updatedSettings.getConfigValue(), SmtpSettings.class); + + assertEquals("updatedUsername", updatedSmtpSettings.getUsername()); + assertEquals("updatedEntity", updatedSmtpSettings.getEmailingEntity()); + } + + @Order(3) + @Test + void testUiThemePreferenceSettings() throws HttpResponseException { + Settings uiThemeSettings = getSystemConfig(SettingsType.CUSTOM_UI_THEME_PREFERENCE); + UiThemePreference uiThemePreference = + JsonUtils.convertValue(uiThemeSettings.getConfigValue(), UiThemePreference.class); + assertEquals("", uiThemePreference.getCustomTheme().getPrimaryColor()); + assertEquals("", uiThemePreference.getCustomLogoConfig().getCustomLogoUrlPath()); + + uiThemePreference.getCustomTheme().setPrimaryColor("#FFFFFF"); + uiThemePreference.getCustomLogoConfig().setCustomLogoUrlPath("http://example.com/logo.png"); + + Settings updatedUiThemeSettings = + new Settings() + .withConfigType(SettingsType.CUSTOM_UI_THEME_PREFERENCE) + .withConfigValue(uiThemePreference); + + updateSystemConfig(updatedUiThemeSettings); + + Settings updatedSettings = getSystemConfig(SettingsType.CUSTOM_UI_THEME_PREFERENCE); + UiThemePreference updatedUiThemePreference = + JsonUtils.convertValue(updatedSettings.getConfigValue(), UiThemePreference.class); + + assertEquals("#FFFFFF", updatedUiThemePreference.getCustomTheme().getPrimaryColor()); + assertEquals( + "http://example.com/logo.png", + updatedUiThemePreference.getCustomLogoConfig().getCustomLogoUrlPath()); + // reset to default + uiThemePreference.getCustomTheme().setPrimaryColor(""); + uiThemePreference.getCustomLogoConfig().setCustomLogoUrlPath(""); + updatedUiThemeSettings = + new Settings() + .withConfigType(SettingsType.CUSTOM_UI_THEME_PREFERENCE) + .withConfigValue(uiThemePreference); + updateSystemConfig(updatedUiThemeSettings); + } + + @Test + void testLoginConfigurationSettings() throws HttpResponseException { + // Retrieve the default login configuration settings + Settings loginSettings = getSystemConfig(SettingsType.LOGIN_CONFIGURATION); + LoginConfiguration loginConfig = + JsonUtils.convertValue(loginSettings.getConfigValue(), LoginConfiguration.class); + + // Assert default values + assertEquals(3, loginConfig.getMaxLoginFailAttempts()); + assertEquals(600, loginConfig.getAccessBlockTime()); + assertEquals(3600, loginConfig.getJwtTokenExpiryTime()); + + // Update login configuration + loginConfig.setMaxLoginFailAttempts(5); + loginConfig.setAccessBlockTime(300); + loginConfig.setJwtTokenExpiryTime(7200); + + Settings updatedLoginSettings = + new Settings() + .withConfigType(SettingsType.LOGIN_CONFIGURATION) + .withConfigValue(loginConfig); + + updateSystemConfig(updatedLoginSettings); + + // Retrieve the updated settings + Settings updatedSettings = getSystemConfig(SettingsType.LOGIN_CONFIGURATION); + LoginConfiguration updatedLoginConfig = + JsonUtils.convertValue(updatedSettings.getConfigValue(), LoginConfiguration.class); + + // Assert updated values + assertEquals(5, updatedLoginConfig.getMaxLoginFailAttempts()); + assertEquals(300, updatedLoginConfig.getAccessBlockTime()); + assertEquals(7200, updatedLoginConfig.getJwtTokenExpiryTime()); + } + + @Test + void testLineageSettings() throws HttpResponseException { + // Retrieve the default lineage settings + Settings lineageSettings = getSystemConfig(SettingsType.LINEAGE_SETTINGS); + LineageSettings lineageConfig = + JsonUtils.convertValue(lineageSettings.getConfigValue(), LineageSettings.class); + + // Assert default values + assertEquals(2, lineageConfig.getUpstreamDepth()); + assertEquals(2, lineageConfig.getDownstreamDepth()); + + // Update lineage settings + lineageConfig.setUpstreamDepth(3); + lineageConfig.setDownstreamDepth(4); + + Settings updatedLineageSettings = + new Settings().withConfigType(SettingsType.LINEAGE_SETTINGS).withConfigValue(lineageConfig); + + updateSystemConfig(updatedLineageSettings); + + // Retrieve the updated settings + Settings updatedSettings = getSystemConfig(SettingsType.LINEAGE_SETTINGS); + LineageSettings updatedLineageConfig = + JsonUtils.convertValue(updatedSettings.getConfigValue(), LineageSettings.class); + + // Assert updated values + assertEquals(3, updatedLineageConfig.getUpstreamDepth()); + assertEquals(4, updatedLineageConfig.getDownstreamDepth()); + } + @Test void globalProfilerConfig(TestInfo test) throws HttpResponseException { // Create a profiler config diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/limitsConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/limitsConfiguration.json index e9ee55cd7c0..17b372354b7 100644 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/limitsConfiguration.json +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/limitsConfiguration.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/entity/configuration/limitsConfiguration.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "FernetConfiguration", + "title": "LimitsConfiguration", "description": "This schema defines the Limits Configuration.", "type": "object", "javaType": "org.openmetadata.schema.configuration.LimitsConfiguration", diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/lineageSettings.json b/openmetadata-spec/src/main/resources/json/schema/configuration/lineageSettings.json new file mode 100644 index 00000000000..64b7664a820 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/lineageSettings.json @@ -0,0 +1,43 @@ +{ + "$id": "https://open-metadata.org/schema/entity/configuration/lineageSettings.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "LineageSettings", + "description": "This schema defines the Lineage Settings.", + "type": "object", + "javaType": "org.openmetadata.schema.api.lineage.LineageSettings", + "definitions": { + "lineageLayer": { + "javaType": "org.openmetadata.schema.api.lineage.LineageLayer", + "description": "Lineage Layers", + "type": "string", + "enum": [ + "EntityLineage", + "ColumnLevelLineage", + "DataObservability" + ], + "default": "EntityLineage" + } + }, + "properties": { + "upstreamDepth": { + "description": "Upstream Depth for Lineage.", + "type": "integer", + "default": 2, + "minimum": 1, + "maximum": 5 + }, + "downstreamDepth": { + "description": "DownStream Depth for Lineage.", + "type": "integer", + "default": 2, + "minimum": 1, + "maximum": 5 + }, + "lineageLayer": { + "description": "Lineage Layer.", + "$ref": "#/definitions/lineageLayer" + } + }, + "required": ["upstreamDepth", "downstreamDepth", "lineageLayer"], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/searchSettings.json b/openmetadata-spec/src/main/resources/json/schema/configuration/searchSettings.json new file mode 100644 index 00000000000..1441f4f472b --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/searchSettings.json @@ -0,0 +1,16 @@ +{ + "$id": "https://open-metadata.org/schema/entity/configuration/searchSettings.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SearchSettings", + "description": "This schema defines the Rbac Search Configuration.", + "type": "object", + "javaType": "org.openmetadata.schema.api.search.SearchSettings", + "properties": { + "enableAccessControl": { + "type": "boolean", + "description": "Flag to enable or disable the RBAC Search Configuration.", + "default": false + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/settings/settings.json b/openmetadata-spec/src/main/resources/json/schema/settings/settings.json index e698d69c494..e2dfcad608a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/settings/settings.json +++ b/openmetadata-spec/src/main/resources/json/schema/settings/settings.json @@ -29,7 +29,8 @@ "slackBot", "slackInstaller", "slackState", - "profilerConfiguration" + "profilerConfiguration", + "lineageSettings" ] } }, @@ -72,6 +73,9 @@ }, { "$ref": "../configuration/profilerConfiguration.json" + }, + { + "$ref": "../configuration/lineageSettings.json" } ] }