yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-01 21:23:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Handle Email From UI (#10906)

Browse Source 
* Handle Email From UI

* Resolved Conflicts

* Fix Failing Tests

* Fix Failing Tests

* Fix Failing checkstyle failing

* Return Masked Password + handle null in case of Saml Missing Config
This commit is contained in:
Mohit Yadav 2023-04-05 10:31:51 +05:30 committed by GitHub
parent 592ae305c3
commit a095d23fb5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 163 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java
View File

@ -82,6 +82,7 @@ import org.openmetadata.service.monitoring.EventMonitorFactory;
import org.openmetadata.service.monitoring.EventMonitorPublisher;
import org.openmetadata.service.resources.CollectionRegistry;
import org.openmetadata.service.resources.databases.DatasourceConfig;
import org.openmetadata.service.resources.settings.SettingsCache;
import org.openmetadata.service.secrets.SecretsManager;
import org.openmetadata.service.secrets.SecretsManagerFactory;
import org.openmetadata.service.secrets.SecretsManagerUpdateService;
@ -103,7 +104,6 @@ import org.openmetadata.service.socket.FeedServlet;
import org.openmetadata.service.socket.OpenMetadataAssetServlet;
import org.openmetadata.service.socket.SocketAddressFilter;
import org.openmetadata.service.socket.WebSocketManager;
import org.openmetadata.service.util.EmailUtil;
import org.openmetadata.service.util.MicrometerBundleSingleton;
/** Main catalog application */
@ -119,12 +119,12 @@ public class OpenMetadataApplication extends Application<OpenMetadataApplication
NoSuchAlgorithmException {
validateConfiguration(catalogConfig);
// init email Util for handling
EmailUtil.initialize(catalogConfig);
ChangeEventConfig.initialize(catalogConfig);
final Jdbi jdbi = createAndSetupJDBI(environment, catalogConfig.getDataSourceFactory());
// Init Settings Cache
SettingsCache.initialize(jdbi.onDemand(CollectionDAO.class), catalogConfig);
// init Secret Manager
final SecretsManager secretsManager =
SecretsManagerFactory.createSecretsManager(

4
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -57,6 +57,7 @@ import org.openmetadata.schema.auth.RefreshToken;
import org.openmetadata.schema.auth.TokenType;
import org.openmetadata.schema.dataInsight.DataInsightChart;
import org.openmetadata.schema.dataInsight.kpi.Kpi;
import org.openmetadata.schema.email.SmtpSettings;
import org.openmetadata.schema.entity.Bot;
import org.openmetadata.schema.entity.Type;
import org.openmetadata.schema.entity.automations.Workflow;
@ -3454,6 +3455,9 @@ public interface CollectionDAO {
case TEST_RESULT_NOTIFICATION_CONFIGURATION:
value = JsonUtils.readValue(json, TestResultNotificationConfiguration.class);
break;
case EMAIL_CONFIGURATION:
value = JsonUtils.readValue(json, SmtpSettings.class);
break;
default:
throw new IllegalArgumentException("Invalid Settings Type " + configType);
}

49
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java
View File

@ -5,10 +5,14 @@ import javax.json.JsonPatch;
import javax.json.JsonValue;
import javax.ws.rs.core.Response;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.email.SmtpSettings;
import org.openmetadata.schema.settings.Settings;
import org.openmetadata.schema.settings.SettingsType;
import org.openmetadata.schema.util.EntitiesCount;
import org.openmetadata.schema.util.ServicesCount;
import org.openmetadata.service.fernet.Fernet;
import org.openmetadata.service.jdbi3.CollectionDAO.SystemDAO;
import org.openmetadata.service.resources.settings.SettingsCache;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.RestUtil;
import org.openmetadata.service.util.ResultList;
@ -47,13 +51,32 @@ public class SystemRepository {
public Settings getConfigWithKey(String key) {
try {
return dao.getConfigWithKey(key);
Settings fetchedSettings = dao.getConfigWithKey(key);
if (fetchedSettings.getConfigType() == SettingsType.EMAIL_CONFIGURATION) {
SmtpSettings emailConfig = (SmtpSettings) fetchedSettings.getConfigValue();
emailConfig.setPassword("***********");
fetchedSettings.setConfigValue(emailConfig);
}
return fetchedSettings;
} catch (Exception ex) {
LOG.error("Error while trying fetch Settings " + ex.getMessage());
}
return null;
}
public Settings getEmailConfigInternal() {
try {
Settings setting = dao.getConfigWithKey(SettingsType.EMAIL_CONFIGURATION.value());
SmtpSettings emailConfig = SystemRepository.decryptSetting((SmtpSettings) setting.getConfigValue());
setting.setConfigValue(emailConfig);
return setting;
} catch (Exception ex) {
LOG.error("Error while trying fetch EMAIL Settings " + ex.getMessage());
}
return null;
}
public Response createOrUpdate(Settings setting) {
Settings oldValue = getConfigWithKey(setting.getConfigType().toString());
try {
@ -95,9 +118,33 @@ public class SystemRepository {
public void updateSetting(Settings setting) {
try {
if (setting.getConfigType() == SettingsType.EMAIL_CONFIGURATION) {
SmtpSettings emailConfig = JsonUtils.convertValue(setting.getConfigValue(), SmtpSettings.class);
if (!Fernet.isTokenized(emailConfig.getPassword())) {
setting.setConfigValue(encryptSetting(emailConfig));
}
// Invalidate Setting
SettingsCache.getInstance().invalidateSettings(SettingsType.EMAIL_CONFIGURATION.value());
}
dao.insertSettings(setting.getConfigType().toString(), JsonUtils.pojoToJson(setting.getConfigValue()));
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
public static SmtpSettings encryptSetting(SmtpSettings decryptedSetting) {
if (Fernet.getInstance().isKeyDefined()) {
String encryptedPwd = Fernet.getInstance().encrypt(decryptedSetting.getPassword());
return decryptedSetting.withPassword(encryptedPwd);
}
return decryptedSetting;
}
public static SmtpSettings decryptSetting(SmtpSettings encryptedSetting) {
if (Fernet.getInstance().isKeyDefined()) {
String decryptedPassword = Fernet.getInstance().decrypt(encryptedSetting.getPassword());
return encryptedSetting.withPassword(decryptedPassword);
}
return encryptedSetting;
}
}

29
openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java
View File

@ -13,14 +13,18 @@
package org.openmetadata.service.resources.settings;
import static org.openmetadata.schema.settings.SettingsType.EMAIL_CONFIGURATION;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import java.util.concurrent.TimeUnit;
import javax.annotation.CheckForNull;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.email.SmtpSettings;
import org.openmetadata.schema.settings.Settings;
import org.openmetadata.schema.settings.SettingsType;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.SystemRepository;
import org.openmetadata.service.util.JsonUtils;
@ -33,12 +37,23 @@ public class SettingsCache {
protected static SystemRepository systemRepository;
// Expected to be called only once from the DefaultAuthorizer
public static void initialize(CollectionDAO dao) {
public static void initialize(CollectionDAO dao, OpenMetadataApplicationConfig config) {
if (!INITIALIZED) {
SETTINGS_CACHE =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new SettingsLoader());
systemRepository = new SystemRepository(dao.systemDAO());
INITIALIZED = true;
createEmailConfiguration(config);
}
}
private static void createEmailConfiguration(OpenMetadataApplicationConfig applicationConfig) {
Settings storedSettings = systemRepository.getConfigWithKey(EMAIL_CONFIGURATION.toString());
if (storedSettings == null) {
// Only in case a config doesn't exist in DB we insert it
SmtpSettings emailConfig = applicationConfig.getSmtpSettings();
Settings setting = new Settings().withConfigType(EMAIL_CONFIGURATION).withConfigValue(emailConfig);
systemRepository.createNewSetting(setting);
}
}
@ -71,9 +86,15 @@ public class SettingsCache {
static class SettingsLoader extends CacheLoader<String, Settings> {
@Override
public Settings load(@CheckForNull String settingsName) {
Settings setting = systemRepository.getConfigWithKey(settingsName);
LOG.info("Loaded Setting {}", setting.getConfigType());
return setting;
Settings fetchedSettings;
if (SettingsType.EMAIL_CONFIGURATION.value().equals(settingsName)) {
fetchedSettings = systemRepository.getEmailConfigInternal();
LOG.info("Loaded Email Setting");
} else {
fetchedSettings = systemRepository.getConfigWithKey(settingsName);
LOG.info("Loaded Setting {}", fetchedSettings.getConfigType());
}
return fetchedSettings;
}
}
}

14
openmetadata-service/src/main/java/org/openmetadata/service/resources/system/ConfigResource.java
View File

@ -72,12 +72,14 @@ public class ConfigResource {
// Remove Ldap Configuration
authenticationConfiguration.setLdapConfiguration(null);
// Remove Saml Fields
SamlSSOClientConfig ssoClientConfig = new SamlSSOClientConfig();
ssoClientConfig.setIdp(
new IdentityProviderConfig()
.withAuthorityUrl(authenticationConfiguration.getSamlConfiguration().getIdp().getAuthorityUrl()));
authenticationConfiguration.setSamlConfiguration(ssoClientConfig);
if (authenticationConfiguration.getSamlConfiguration() != null) {
// Remove Saml Fields
SamlSSOClientConfig ssoClientConfig = new SamlSSOClientConfig();
ssoClientConfig.setIdp(
new IdentityProviderConfig()
.withAuthorityUrl(authenticationConfiguration.getSamlConfiguration().getIdp().getAuthorityUrl()));
authenticationConfiguration.setSamlConfiguration(ssoClientConfig);
}
}
return authenticationConfiguration;
}

4
openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java
View File

@ -65,10 +65,10 @@ public class SystemResource {
@SuppressWarnings("unused") // Method used for reflection
public void initialize(OpenMetadataApplicationConfig config) throws IOException {
initSettings();
initSettings(config);
}
private void initSettings() throws IOException {
private void initSettings(OpenMetadataApplicationConfig applicationConfig) throws IOException {
List<String> jsonDataFiles = EntityUtil.getJsonDataResources(".*json/data/settings/settingsData.json$");
if (jsonDataFiles.size() != 1) {
LOG.warn("Invalid number of jsonDataFiles {}. Only one expected.", jsonDataFiles.size());

129
openmetadata-service/src/main/java/org/openmetadata/service/util/EmailUtil.java
View File

@ -31,9 +31,10 @@ import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.email.SmtpSettings;
import org.openmetadata.schema.entity.feed.Thread;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.settings.SettingsType;
import org.openmetadata.schema.tests.type.TestCaseResult;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.events.subscription.emailAlert.EmailMessage;
import org.openmetadata.service.resources.settings.SettingsCache;
import org.simplejavamail.api.email.Email;
import org.simplejavamail.api.email.EmailPopulatingBuilder;
import org.simplejavamail.api.mailer.Mailer;
@ -76,61 +77,55 @@ public class EmailUtil {
public static final String TASK_NOTIFICATION_TEMPLATE = "taskAssignment.ftl";
public static final String TEST_NOTIFICATION_TEMPLATE = "testResultStatus.ftl";
private static EmailUtil INSTANCE;
private static SmtpSettings DEFAULT_SMTP_SETTINGS;
private static SmtpSettings STORED_SMTP_SETTINGS;
private static Mailer MAILER;
private static Configuration TEMPLATE_CONFIGURATION;
private static volatile boolean INITIALIZED = false;
public static void initialize(OpenMetadataApplicationConfig config) {
if (!INITIALIZED) {
if (config.getSmtpSettings() != null && config.getSmtpSettings().getEnableSmtpServer()) {
try {
DEFAULT_SMTP_SETTINGS = config.getSmtpSettings();
MAILER = createMailer(DEFAULT_SMTP_SETTINGS);
TEMPLATE_CONFIGURATION = new Configuration(VERSION_2_3_28);
LOG.info("Email Util cache is initialized");
} catch (Exception ex) {
LOG.warn("[MAILER] Smtp Configurations are missing : Reason {} ", ex.getMessage(), ex);
}
} else {
DEFAULT_SMTP_SETTINGS = new SmtpSettings();
}
INSTANCE = new EmailUtil();
INITIALIZED = true;
} else {
INITIALIZED = false;
LOG.info("Email Util is already initialized");
private EmailUtil() {
try {
STORED_SMTP_SETTINGS = getSmtpSettings();
MAILER = createMailer(STORED_SMTP_SETTINGS);
TEMPLATE_CONFIGURATION = new Configuration(VERSION_2_3_28);
LOG.info("Email Util cache is initialized");
} catch (Exception ex) {
LOG.warn("[MAILER] Smtp Configurations are missing : Reason {} ", ex.getMessage(), ex);
}
}
private static Mailer createMailer(SmtpSettings smtpServerSettings) {
TransportStrategy strategy;
switch (smtpServerSettings.getTransportationStrategy()) {
case SMPTS:
strategy = SMTPS;
break;
case SMTP_TLS:
strategy = SMTP_TLS;
break;
default:
strategy = SMTP;
break;
if (smtpServerSettings.getEnableSmtpServer()) {
TransportStrategy strategy;
switch (smtpServerSettings.getTransportationStrategy()) {
case SMPTS:
strategy = SMTPS;
break;
case SMTP_TLS:
strategy = SMTP_TLS;
break;
default:
strategy = SMTP;
break;
}
return MailerBuilder.withSMTPServer(
smtpServerSettings.getServerEndpoint(),
smtpServerSettings.getServerPort(),
smtpServerSettings.getUsername(),
smtpServerSettings.getPassword())
.withTransportStrategy(strategy)
.buildMailer();
}
return MailerBuilder.withSMTPServer(
smtpServerSettings.getServerEndpoint(),
smtpServerSettings.getServerPort(),
smtpServerSettings.getUsername(),
smtpServerSettings.getPassword())
.withTransportStrategy(strategy)
.buildMailer();
return null;
}
public static EmailUtil getInstance() {
if (INSTANCE == null) {
INSTANCE = new EmailUtil();
}
return INSTANCE;
}
public void sendAccountStatus(User user, String action, String status) throws IOException, TemplateException {
if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
if (getSmtpSettings().getEnableSmtpServer()) {
Map<String, String> templatePopulator = new HashMap<>();
templatePopulator.put(ENTITY, getEmailingEntity());
templatePopulator.put(SUPPORT_URL, getSupportUrl());
@ -147,7 +142,7 @@ public class EmailUtil {
}
public void sendEmailVerification(String emailVerificationLink, User user) throws IOException, TemplateException {
if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
if (getSmtpSettings().getEnableSmtpServer()) {
Map<String, String> templatePopulator = new HashMap<>();
templatePopulator.put(ENTITY, getEmailingEntity());
templatePopulator.put(SUPPORT_URL, getSupportUrl());
@ -165,7 +160,7 @@ public class EmailUtil {
public void sendPasswordResetLink(String passwordResetLink, User user, String subject, String templateFilePath)
throws IOException, TemplateException {
if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
if (getSmtpSettings().getEnableSmtpServer()) {
Map<String, String> templatePopulator = new HashMap<>();
templatePopulator.put(ENTITY, getEmailingEntity());
templatePopulator.put(SUPPORT_URL, getSupportUrl());
@ -180,7 +175,7 @@ public class EmailUtil {
public void sendTaskAssignmentNotificationToUser(
String assigneeName, String email, String taskLink, Thread thread, String subject, String templateFilePath)
throws IOException, TemplateException {
if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
if (getSmtpSettings().getEnableSmtpServer()) {
Map<String, String> templatePopulator = new HashMap<>();
templatePopulator.put("assignee", assigneeName);
templatePopulator.put("createdBy", thread.getCreatedBy());
@ -203,7 +198,7 @@ public class EmailUtil {
String subject,
String templateFilePath)
throws IOException, TemplateException {
if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
if (getSmtpSettings().getEnableSmtpServer()) {
Map<String, String> templatePopulator = new HashMap<>();
templatePopulator.put("receiverName", email.split("@")[0]);
templatePopulator.put("testResultName", testCaseName);
@ -219,11 +214,11 @@ public class EmailUtil {
public void sendMail(
String subject, Map<String, String> model, String to, String baseTemplatePackage, String templatePath)
throws IOException, TemplateException {
if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
if (getSmtpSettings().getEnableSmtpServer()) {
EmailPopulatingBuilder emailBuilder = EmailBuilder.startingBlank();
emailBuilder.withSubject(subject);
emailBuilder.to(to);
emailBuilder.from(DEFAULT_SMTP_SETTINGS.getSenderMail());
emailBuilder.from(getSmtpSettings().getSenderMail());
TEMPLATE_CONFIGURATION.setClassForTemplateLoading(getClass(), baseTemplatePackage);
Template template = TEMPLATE_CONFIGURATION.getTemplate(templatePath);
@ -238,17 +233,17 @@ public class EmailUtil {
}
public void sendMail(Email email) {
if (MAILER != null && DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
if (MAILER != null && getSmtpSettings().getEnableSmtpServer()) {
MAILER.sendMail(email, true);
}
}
public String buildBaseUrl(URI uri) {
try {
if (CommonUtil.nullOrEmpty(DEFAULT_SMTP_SETTINGS.getOpenMetadataUrl())) {
if (CommonUtil.nullOrEmpty(getSmtpSettings().getOpenMetadataUrl())) {
return String.format("%s://%s", uri.getScheme(), uri.getHost());
} else {
URI serverUrl = new URI(DEFAULT_SMTP_SETTINGS.getOpenMetadataUrl());
URI serverUrl = new URI(getSmtpSettings().getOpenMetadataUrl());
return String.format("%s://%s", serverUrl.getScheme(), serverUrl.getHost());
}
} catch (Exception ex) {
@ -257,7 +252,7 @@ public class EmailUtil {
}
public static void sendInviteMailToAdmin(User user, String pwd) {
if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
if (getSmtpSettings().getEnableSmtpServer()) {
Map<String, String> templatePopulator = new HashMap<>();
templatePopulator.put(EmailUtil.ENTITY, EmailUtil.getInstance().getEmailingEntity());
templatePopulator.put(EmailUtil.SUPPORT_URL, EmailUtil.getInstance().getSupportUrl());
@ -279,7 +274,7 @@ public class EmailUtil {
}
public static void sendChangeEventMail(String receiverMail, EmailMessage emailMessaged) {
if (DEFAULT_SMTP_SETTINGS.getEnableSmtpServer()) {
if (getSmtpSettings().getEnableSmtpServer()) {
Map<String, String> templatePopulator = new HashMap<>();
templatePopulator.put(EmailUtil.USERNAME, receiverMail.split("@")[0]);
templatePopulator.put("updatedBy", emailMessaged.getUpdatedBy());
@ -309,42 +304,52 @@ public class EmailUtil {
}
private String getEmailVerificationSubject() {
return String.format(EMAIL_VERIFICATION_SUBJECT, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
return String.format(EMAIL_VERIFICATION_SUBJECT, getSmtpSettings().getEmailingEntity());
}
public String getPasswordResetSubject() {
return String.format(PASSWORD_RESET_SUBJECT, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
return String.format(PASSWORD_RESET_SUBJECT, getSmtpSettings().getEmailingEntity());
}
private String getAccountStatusChangeSubject() {
return String.format(ACCOUNT_STATUS_SUBJECT, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
return String.format(ACCOUNT_STATUS_SUBJECT, getSmtpSettings().getEmailingEntity());
}
public String getEmailInviteSubject() {
return String.format(INVITE_SUBJECT, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
return String.format(INVITE_SUBJECT, getSmtpSettings().getEmailingEntity());
}
public String getChangeEventTemplate() {
return String.format(CHANGE_EVENT_UPDATE, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
return String.format(CHANGE_EVENT_UPDATE, getSmtpSettings().getEmailingEntity());
}
public String getTaskAssignmentSubject() {
return String.format(TASK_SUBJECT, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
return String.format(TASK_SUBJECT, getSmtpSettings().getEmailingEntity());
}
public String getTestResultSubject() {
return String.format(TEST_SUBJECT, DEFAULT_SMTP_SETTINGS.getEmailingEntity());
return String.format(TEST_SUBJECT, getSmtpSettings().getEmailingEntity());
}
public String getEmailingEntity() {
return DEFAULT_SMTP_SETTINGS.getEmailingEntity();
return getSmtpSettings().getEmailingEntity();
}
public String getSupportUrl() {
return DEFAULT_SMTP_SETTINGS.getSupportUrl();
return getSmtpSettings().getSupportUrl();
}
public String getOMUrl() {
return DEFAULT_SMTP_SETTINGS.getOpenMetadataUrl();
return getSmtpSettings().getOpenMetadataUrl();
}
private static SmtpSettings getSmtpSettings() {
SmtpSettings emailConfig =
SettingsCache.getInstance().getSetting(SettingsType.EMAIL_CONFIGURATION, SmtpSettings.class);
if (!emailConfig.equals(STORED_SMTP_SETTINGS)) {
STORED_SMTP_SETTINGS = emailConfig;
MAILER = createMailer(emailConfig);
}
return emailConfig;
}
}

6
openmetadata-spec/src/main/resources/json/schema/settings/settings.json
View File

@ -23,7 +23,8 @@
"sandboxModeEnabled",
"slackChat",
"taskNotificationConfiguration",
"testResultNotificationConfiguration"
"testResultNotificationConfiguration",
"emailConfiguration"
]
}
},
@ -57,6 +58,9 @@
},
{
"$ref": "../configuration/taskNotificationConfiguration.json"
},
{
"$ref": "../email/smtpSettings.json"
}
]