Handle Email From UI (#10906)

* Handle Email From UI

* Resolved Conflicts

* Fix Failing Tests

* Fix Failing Tests

* Fix Failing checkstyle failing

* Return Masked Password + handle null in case of Saml Missing Config

openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java

@@ -82,6 +82,7 @@ import org.openmetadata.service.monitoring.EventMonitorFactory;
 import org.openmetadata.service.monitoring.EventMonitorPublisher;
 import org.openmetadata.service.resources.CollectionRegistry;
 import org.openmetadata.service.resources.databases.DatasourceConfig;
+import org.openmetadata.service.resources.settings.SettingsCache;
 import org.openmetadata.service.secrets.SecretsManager;
 import org.openmetadata.service.secrets.SecretsManagerFactory;
 import org.openmetadata.service.secrets.SecretsManagerUpdateService;
@@ -103,7 +104,6 @@ import org.openmetadata.service.socket.FeedServlet;
 import org.openmetadata.service.socket.OpenMetadataAssetServlet;
 import org.openmetadata.service.socket.SocketAddressFilter;
 import org.openmetadata.service.socket.WebSocketManager;
-import org.openmetadata.service.util.EmailUtil;
 import org.openmetadata.service.util.MicrometerBundleSingleton;
 
 /** Main catalog application */
@@ -119,12 +119,12 @@ public class OpenMetadataApplication extends Application<OpenMetadataApplication
           NoSuchAlgorithmException {
     validateConfiguration(catalogConfig);
 
-    // init email Util for handling
-    EmailUtil.initialize(catalogConfig);
-
     ChangeEventConfig.initialize(catalogConfig);
     final Jdbi jdbi = createAndSetupJDBI(environment, catalogConfig.getDataSourceFactory());
 
+    // Init Settings Cache
+    SettingsCache.initialize(jdbi.onDemand(CollectionDAO.class), catalogConfig);
+
     // init Secret Manager
     final SecretsManager secretsManager =
         SecretsManagerFactory.createSecretsManager(

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java

@@ -57,6 +57,7 @@ import org.openmetadata.schema.auth.RefreshToken;
 import org.openmetadata.schema.auth.TokenType;
 import org.openmetadata.schema.dataInsight.DataInsightChart;
 import org.openmetadata.schema.dataInsight.kpi.Kpi;
+import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.entity.Bot;
 import org.openmetadata.schema.entity.Type;
 import org.openmetadata.schema.entity.automations.Workflow;
@@ -3454,6 +3455,9 @@ public interface CollectionDAO {
           case TEST_RESULT_NOTIFICATION_CONFIGURATION:
             value = JsonUtils.readValue(json, TestResultNotificationConfiguration.class);
             break;
+          case EMAIL_CONFIGURATION:
+            value = JsonUtils.readValue(json, SmtpSettings.class);
+            break;
           default:
             throw new IllegalArgumentException("Invalid Settings Type " + configType);
         }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java

@@ -5,10 +5,14 @@ import javax.json.JsonPatch;
 import javax.json.JsonValue;
 import javax.ws.rs.core.Response;
 import lombok.extern.slf4j.Slf4j;
+import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.settings.Settings;
+import org.openmetadata.schema.settings.SettingsType;
 import org.openmetadata.schema.util.EntitiesCount;
 import org.openmetadata.schema.util.ServicesCount;
+import org.openmetadata.service.fernet.Fernet;
 import org.openmetadata.service.jdbi3.CollectionDAO.SystemDAO;
+import org.openmetadata.service.resources.settings.SettingsCache;
 import org.openmetadata.service.util.JsonUtils;
 import org.openmetadata.service.util.RestUtil;
 import org.openmetadata.service.util.ResultList;
@@ -47,13 +51,32 @@ public class SystemRepository {
 
   public Settings getConfigWithKey(String key) {
     try {
-      return dao.getConfigWithKey(key);
+      Settings fetchedSettings = dao.getConfigWithKey(key);
+      if (fetchedSettings.getConfigType() == SettingsType.EMAIL_CONFIGURATION) {
+        SmtpSettings emailConfig = (SmtpSettings) fetchedSettings.getConfigValue();
+        emailConfig.setPassword("***********");
+        fetchedSettings.setConfigValue(emailConfig);
+      }
+      return fetchedSettings;
+
     } catch (Exception ex) {
       LOG.error("Error while trying fetch Settings " + ex.getMessage());
     }
     return null;
   }
 
+  public Settings getEmailConfigInternal() {
+    try {
+      Settings setting = dao.getConfigWithKey(SettingsType.EMAIL_CONFIGURATION.value());
+      SmtpSettings emailConfig = SystemRepository.decryptSetting((SmtpSettings) setting.getConfigValue());
+      setting.setConfigValue(emailConfig);
+      return setting;
+    } catch (Exception ex) {
+      LOG.error("Error while trying fetch EMAIL Settings " + ex.getMessage());
+    }
+    return null;
+  }
+
   public Response createOrUpdate(Settings setting) {
     Settings oldValue = getConfigWithKey(setting.getConfigType().toString());
     try {
@@ -95,9 +118,33 @@ public class SystemRepository {
 
   public void updateSetting(Settings setting) {
     try {
+      if (setting.getConfigType() == SettingsType.EMAIL_CONFIGURATION) {
+        SmtpSettings emailConfig = JsonUtils.convertValue(setting.getConfigValue(), SmtpSettings.class);
+        if (!Fernet.isTokenized(emailConfig.getPassword())) {
+          setting.setConfigValue(encryptSetting(emailConfig));
+        }
+        // Invalidate Setting
+        SettingsCache.getInstance().invalidateSettings(SettingsType.EMAIL_CONFIGURATION.value());
+      }
       dao.insertSettings(setting.getConfigType().toString(), JsonUtils.pojoToJson(setting.getConfigValue()));
     } catch (Exception ex) {
       throw new RuntimeException(ex);
     }
   }
+
+  public static SmtpSettings encryptSetting(SmtpSettings decryptedSetting) {
+    if (Fernet.getInstance().isKeyDefined()) {
+      String encryptedPwd = Fernet.getInstance().encrypt(decryptedSetting.getPassword());
+      return decryptedSetting.withPassword(encryptedPwd);
+    }
+    return decryptedSetting;
+  }
+
+  public static SmtpSettings decryptSetting(SmtpSettings encryptedSetting) {
+    if (Fernet.getInstance().isKeyDefined()) {
+      String decryptedPassword = Fernet.getInstance().decrypt(encryptedSetting.getPassword());
+      return encryptedSetting.withPassword(decryptedPassword);
+    }
+    return encryptedSetting;
+  }
 }

openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java

@@ -13,14 +13,18 @@
 
 package org.openmetadata.service.resources.settings;
 
+import static org.openmetadata.schema.settings.SettingsType.EMAIL_CONFIGURATION;
+
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import java.util.concurrent.TimeUnit;
 import javax.annotation.CheckForNull;
 import lombok.extern.slf4j.Slf4j;
+import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.settings.Settings;
 import org.openmetadata.schema.settings.SettingsType;
+import org.openmetadata.service.OpenMetadataApplicationConfig;
 import org.openmetadata.service.jdbi3.CollectionDAO;
 import org.openmetadata.service.jdbi3.SystemRepository;
 import org.openmetadata.service.util.JsonUtils;
@@ -33,12 +37,23 @@ public class SettingsCache {
   protected static SystemRepository systemRepository;
 
   // Expected to be called only once from the DefaultAuthorizer
-  public static void initialize(CollectionDAO dao) {
+  public static void initialize(CollectionDAO dao, OpenMetadataApplicationConfig config) {
     if (!INITIALIZED) {
       SETTINGS_CACHE =
           CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new SettingsLoader());
       systemRepository = new SystemRepository(dao.systemDAO());
       INITIALIZED = true;
+      createEmailConfiguration(config);
+    }
+  }
+
+  private static void createEmailConfiguration(OpenMetadataApplicationConfig applicationConfig) {
+    Settings storedSettings = systemRepository.getConfigWithKey(EMAIL_CONFIGURATION.toString());
+    if (storedSettings == null) {
+      // Only in case a config doesn't exist in DB we insert it
+      SmtpSettings emailConfig = applicationConfig.getSmtpSettings();
+      Settings setting = new Settings().withConfigType(EMAIL_CONFIGURATION).withConfigValue(emailConfig);
+      systemRepository.createNewSetting(setting);
     }
   }
 
@@ -71,9 +86,15 @@ public class SettingsCache {
   static class SettingsLoader extends CacheLoader<String, Settings> {
     @Override
     public Settings load(@CheckForNull String settingsName) {
-      Settings setting = systemRepository.getConfigWithKey(settingsName);
+      Settings fetchedSettings;
-      LOG.info("Loaded Setting {}", setting.getConfigType());
+      if (SettingsType.EMAIL_CONFIGURATION.value().equals(settingsName)) {
-      return setting;
+        fetchedSettings = systemRepository.getEmailConfigInternal();
+        LOG.info("Loaded Email Setting");
+      } else {
+        fetchedSettings = systemRepository.getConfigWithKey(settingsName);
+        LOG.info("Loaded Setting {}", fetchedSettings.getConfigType());
+      }
+      return fetchedSettings;
     }
   }
 }

openmetadata-service/src/main/java/org/openmetadata/service/resources/system/ConfigResource.java

@@ -72,6 +72,7 @@ public class ConfigResource {
       // Remove Ldap Configuration
       authenticationConfiguration.setLdapConfiguration(null);
 
+      if (authenticationConfiguration.getSamlConfiguration() != null) {
         // Remove Saml Fields
         SamlSSOClientConfig ssoClientConfig = new SamlSSOClientConfig();
         ssoClientConfig.setIdp(
@@ -79,6 +80,7 @@ public class ConfigResource {
                 .withAuthorityUrl(authenticationConfiguration.getSamlConfiguration().getIdp().getAuthorityUrl()));
         authenticationConfiguration.setSamlConfiguration(ssoClientConfig);
       }
+    }
     return authenticationConfiguration;
   }
 

openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java

@@ -65,10 +65,10 @@ public class SystemResource {
 
   @SuppressWarnings("unused") // Method used for reflection
   public void initialize(OpenMetadataApplicationConfig config) throws IOException {
-    initSettings();
+    initSettings(config);
   }
 
-  private void initSettings() throws IOException {
+  private void initSettings(OpenMetadataApplicationConfig applicationConfig) throws IOException {
     List<String> jsonDataFiles = EntityUtil.getJsonDataResources(".*json/data/settings/settingsData.json$");
     if (jsonDataFiles.size() != 1) {
       LOG.warn("Invalid number of jsonDataFiles {}. Only one expected.", jsonDataFiles.size());

openmetadata-service/src/main/java/org/openmetadata/service/util/EmailUtil.java

@@ -31,9 +31,10 @@ import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.entity.feed.Thread;
 import org.openmetadata.schema.entity.teams.User;
+import org.openmetadata.schema.settings.SettingsType;
 import org.openmetadata.schema.tests.type.TestCaseResult;
-import org.openmetadata.service.OpenMetadataApplicationConfig;
 import org.openmetadata.service.events.subscription.emailAlert.EmailMessage;
+import org.openmetadata.service.resources.settings.SettingsCache;
 import org.simplejavamail.api.email.Email;
 import org.simplejavamail.api.email.EmailPopulatingBuilder;
 import org.simplejavamail.api.mailer.Mailer;
@@ -76,34 +77,23 @@ public class EmailUtil {
   public static final String TASK_NOTIFICATION_TEMPLATE = "taskAssignment.ftl";
   public static final String TEST_NOTIFICATION_TEMPLATE = "testResultStatus.ftl";
   private static EmailUtil INSTANCE;
-  private static SmtpSettings DEFAULT_SMTP_SETTINGS;
+  private static SmtpSettings STORED_SMTP_SETTINGS;
   private static Mailer MAILER;
   private static Configuration TEMPLATE_CONFIGURATION;
-  private static volatile boolean INITIALIZED = false;
 
-  public static void initialize(OpenMetadataApplicationConfig config) {
+  private EmailUtil() {
-    if (!INITIALIZED) {
-      if (config.getSmtpSettings() != null && config.getSmtpSettings().getEnableSmtpServer()) {
     try {
-          DEFAULT_SMTP_SETTINGS = config.getSmtpSettings();
+      STORED_SMTP_SETTINGS = getSmtpSettings();
-          MAILER = createMailer(DEFAULT_SMTP_SETTINGS);
+      MAILER = createMailer(STORED_SMTP_SETTINGS);
       TEMPLATE_CONFIGURATION = new Configuration(VERSION_2_3_28);
       LOG.info("Email Util cache is initialized");
     } catch (Exception ex) {
       LOG.warn("[MAILER] Smtp Configurations are missing : Reason {} ", ex.getMessage(), ex);
     }
-      } else {
-        DEFAULT_SMTP_SETTINGS = new SmtpSettings();
-      }
-      INSTANCE = new EmailUtil();
-      INITIALIZED = true;
-    } else {
-      INITIALIZED = false;
-      LOG.info("Email Util is already initialized");
-    }
   }
 
   private static Mailer createMailer(SmtpSettings smtpServerSettings) {
+    if (smtpServerSettings.getEnableSmtpServer()) {
       TransportStrategy strategy;
       switch (smtpServerSettings.getTransportationStrategy()) {
         case SMPTS:
@@ -124,13 +114,18 @@ public class EmailUtil {
           .withTransportStrategy(strategy)
           .buildMailer();
     }
+    return null;
+  }
 
   public static EmailUtil getInstance() {
+    if (INSTANCE == null) {
+      INSTANCE = new EmailUtil();
+    }
     return INSTANCE;
   }
 
   public void sendAccountStatus(User user, String action, String status) throws IOException, TemplateException {
-    if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
+    if (getSmtpSettings().getEnableSmtpServer()) {
       Map<String, String> templatePopulator = new HashMap<>();
       templatePopulator.put(ENTITY, getEmailingEntity());
       templatePopulator.put(SUPPORT_URL, getSupportUrl());
@@ -147,7 +142,7 @@ public class EmailUtil {
   }
 
   public void sendEmailVerification(String emailVerificationLink, User user) throws IOException, TemplateException {
-    if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
+    if (getSmtpSettings().getEnableSmtpServer()) {
       Map<String, String> templatePopulator = new HashMap<>();
       templatePopulator.put(ENTITY, getEmailingEntity());
       templatePopulator.put(SUPPORT_URL, getSupportUrl());
@@ -165,7 +160,7 @@ public class EmailUtil {
 
   public void sendPasswordResetLink(String passwordResetLink, User user, String subject, String templateFilePath)
       throws IOException, TemplateException {
-    if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
+    if (getSmtpSettings().getEnableSmtpServer()) {
       Map<String, String> templatePopulator = new HashMap<>();
       templatePopulator.put(ENTITY, getEmailingEntity());
       templatePopulator.put(SUPPORT_URL, getSupportUrl());
@@ -180,7 +175,7 @@ public class EmailUtil {
   public void sendTaskAssignmentNotificationToUser(
       String assigneeName, String email, String taskLink, Thread thread, String subject, String templateFilePath)
       throws IOException, TemplateException {
-    if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
+    if (getSmtpSettings().getEnableSmtpServer()) {
       Map<String, String> templatePopulator = new HashMap<>();
       templatePopulator.put("assignee", assigneeName);
       templatePopulator.put("createdBy", thread.getCreatedBy());
@@ -203,7 +198,7 @@ public class EmailUtil {
       String subject,
       String templateFilePath)
       throws IOException, TemplateException {
-    if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
+    if (getSmtpSettings().getEnableSmtpServer()) {
       Map<String, String> templatePopulator = new HashMap<>();
       templatePopulator.put("receiverName", email.split("@")[0]);
       templatePopulator.put("testResultName", testCaseName);
@@ -219,11 +214,11 @@ public class EmailUtil {
   public void sendMail(
       String subject, Map<String, String> model, String to, String baseTemplatePackage, String templatePath)
       throws IOException, TemplateException {
-    if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
+    if (getSmtpSettings().getEnableSmtpServer()) {
       EmailPopulatingBuilder emailBuilder = EmailBuilder.startingBlank();
       emailBuilder.withSubject(subject);
       emailBuilder.to(to);
-      emailBuilder.from(DEFAULT_SMTP_SETTINGS.getSenderMail());
+      emailBuilder.from(getSmtpSettings().getSenderMail());
 
       TEMPLATE_CONFIGURATION.setClassForTemplateLoading(getClass(), baseTemplatePackage);
       Template template = TEMPLATE_CONFIGURATION.getTemplate(templatePath);
@@ -238,17 +233,17 @@ public class EmailUtil {
   }
 
   public void sendMail(Email email) {
-    if (MAILER != null && DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
+    if (MAILER != null && getSmtpSettings().getEnableSmtpServer()) {
       MAILER.sendMail(email, true);
     }
   }
 
   public String buildBaseUrl(URI uri) {
     try {
-      if (CommonUtil.nullOrEmpty(DEFAULT_SMTP_SETTINGS.getOpenMetadataUrl())) {
+      if (CommonUtil.nullOrEmpty(getSmtpSettings().getOpenMetadataUrl())) {
         return String.format("%s://%s", uri.getScheme(), uri.getHost());
       } else {
-        URI serverUrl = new URI(DEFAULT_SMTP_SETTINGS.getOpenMetadataUrl());
+        URI serverUrl = new URI(getSmtpSettings().getOpenMetadataUrl());
         return String.format("%s://%s", serverUrl.getScheme(), serverUrl.getHost());
       }
     } catch (Exception ex) {
@@ -257,7 +252,7 @@ public class EmailUtil {
   }
 
   public static void sendInviteMailToAdmin(User user, String pwd) {
-    if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
+    if (getSmtpSettings().getEnableSmtpServer()) {
       Map<String, String> templatePopulator = new HashMap<>();
       templatePopulator.put(EmailUtil.ENTITY, EmailUtil.getInstance().getEmailingEntity());
       templatePopulator.put(EmailUtil.SUPPORT_URL, EmailUtil.getInstance().getSupportUrl());
@@ -279,7 +274,7 @@ public class EmailUtil {
   }
 
   public static void sendChangeEventMail(String receiverMail, EmailMessage emailMessaged) {
-    if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
+    if (getSmtpSettings().getEnableSmtpServer()) {
       Map<String, String> templatePopulator = new HashMap<>();
       templatePopulator.put(EmailUtil.USERNAME, receiverMail.split("@")[0]);
       templatePopulator.put("updatedBy", emailMessaged.getUpdatedBy());
@@ -309,42 +304,52 @@ public class EmailUtil {
   }
 
   private String getEmailVerificationSubject() {
-    return String.format(EMAIL_VERIFICATION_SUBJECT, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
+    return String.format(EMAIL_VERIFICATION_SUBJECT, getSmtpSettings().getEmailingEntity());
   }
 
   public String getPasswordResetSubject() {
-    return String.format(PASSWORD_RESET_SUBJECT, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
+    return String.format(PASSWORD_RESET_SUBJECT, getSmtpSettings().getEmailingEntity());
   }
 
   private String getAccountStatusChangeSubject() {
-    return String.format(ACCOUNT_STATUS_SUBJECT, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
+    return String.format(ACCOUNT_STATUS_SUBJECT, getSmtpSettings().getEmailingEntity());
   }
 
   public String getEmailInviteSubject() {
-    return String.format(INVITE_SUBJECT, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
+    return String.format(INVITE_SUBJECT, getSmtpSettings().getEmailingEntity());
   }
 
   public String getChangeEventTemplate() {
-    return String.format(CHANGE_EVENT_UPDATE, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
+    return String.format(CHANGE_EVENT_UPDATE, getSmtpSettings().getEmailingEntity());
   }
 
   public String getTaskAssignmentSubject() {
-    return String.format(TASK_SUBJECT, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
+    return String.format(TASK_SUBJECT, getSmtpSettings().getEmailingEntity());
   }
 
   public String getTestResultSubject() {
-    return String.format(TEST_SUBJECT, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
+    return String.format(TEST_SUBJECT, getSmtpSettings().getEmailingEntity());
   }
 
   public String getEmailingEntity() {
-    return DEFAULT_SMTP_SETTINGS.getEmailingEntity();
+    return getSmtpSettings().getEmailingEntity();
   }
 
   public String getSupportUrl() {
-    return DEFAULT_SMTP_SETTINGS.getSupportUrl();
+    return getSmtpSettings().getSupportUrl();
   }
 
   public String getOMUrl() {
-    return DEFAULT_SMTP_SETTINGS.getOpenMetadataUrl();
+    return getSmtpSettings().getOpenMetadataUrl();
+  }
+
+  private static SmtpSettings getSmtpSettings() {
+    SmtpSettings emailConfig =
+        SettingsCache.getInstance().getSetting(SettingsType.EMAIL_CONFIGURATION, SmtpSettings.class);
+    if (!emailConfig.equals(STORED_SMTP_SETTINGS)) {
+      STORED_SMTP_SETTINGS = emailConfig;
+      MAILER = createMailer(emailConfig);
+    }
+    return emailConfig;
   }
 }

openmetadata-spec/src/main/resources/json/schema/settings/settings.json

@@ -23,7 +23,8 @@
         "sandboxModeEnabled",
         "slackChat",
         "taskNotificationConfiguration",
-        "testResultNotificationConfiguration"
+        "testResultNotificationConfiguration",
+        "emailConfiguration"
       ]
     }
   },
@@ -57,6 +58,9 @@
         },
         {
           "$ref": "../configuration/taskNotificationConfiguration.json"
+        },
+        {
+          "$ref": "../email/smtpSettings.json"
         }
 
       ]