New Email Templates (OSS) (#17606)

* Set new emailTemplates for OSS. Minor other improvements.

* Set new emailTemplates for OSS. Minor other improvements.

* fix template.

* set changeCount in the templates.

* fix changeCount calculation

* migrations.

openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java

@@ -123,12 +123,16 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
         try {
           DataInsightTotalAssetTemplate totalAssetTemplate =
               createTotalAssetTemplate(searchClient, team.getName(), timeConfig, contextData);
+
           DataInsightDescriptionAndOwnerTemplate descriptionTemplate =
               createDescriptionTemplate(searchClient, team.getName(), timeConfig, contextData);
+
           DataInsightDescriptionAndOwnerTemplate ownershipTemplate =
               createOwnershipTemplate(searchClient, team.getName(), timeConfig, contextData);
+
           DataInsightDescriptionAndOwnerTemplate tierTemplate =
               createTierTemplate(searchClient, team.getName(), timeConfig, contextData);
+
           EmailUtil.sendDataInsightEmailNotificationToUser(
               emails,
               getMonthAndDateFromEpoch(timeConfig.startTime()),
@@ -213,13 +217,20 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
     dateWithCount.forEach((key, value) -> dateMap.put(key, value.intValue()));
     processDateMapToNormalize(dateMap);
 
+    int changeInTotalAssets = (int) (currentCount - previousCount);
+
     if (previousCount == 0D) {
       // it should be undefined
       return new DataInsightTotalAssetTemplate(
-          String.valueOf(currentCount.intValue()), 0D, timeConfig.numberOfDaysChange(), dateMap);
+          String.valueOf(currentCount.intValue()),
+          currentCount.intValue(),
+          0d,
+          timeConfig.numberOfDaysChange(),
+          dateMap);
     } else {
       return new DataInsightTotalAssetTemplate(
           String.valueOf(currentCount.intValue()),
+          changeInTotalAssets,
           ((currentCount - previousCount) / previousCount) * 100,
           timeConfig.numberOfDaysChange(),
           dateMap);
@@ -264,10 +275,13 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
       currentPercentCompleted = (currentCompletedDescription / currentTotalAssetCount) * 100;
     }
 
+    int changeCount = (int) (currentCompletedDescription - previousCompletedDescription);
+
     return getTemplate(
         DataInsightDescriptionAndOwnerTemplate.MetricType.DESCRIPTION,
         "percentage_of_data_asset_with_description_kpi",
         currentPercentCompleted,
+        changeCount,
         currentPercentCompleted - previousPercentCompleted,
         currentCompletedDescription.intValue(),
         timeConfig.numberOfDaysChange(),
@@ -312,10 +326,13 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
       currentPercentCompleted = (currentHasOwner / currentTotalAssetCount) * 100;
     }
 
+    int changeCount = (int) (currentHasOwner - previousHasOwner);
+
     return getTemplate(
         DataInsightDescriptionAndOwnerTemplate.MetricType.OWNER,
         "percentage_of_data_asset_with_owner_kpi",
         currentPercentCompleted,
+        changeCount,
         currentPercentCompleted - previousPercentCompleted,
         currentHasOwner.intValue(),
         timeConfig.numberOfDaysChange(),
@@ -358,6 +375,8 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
       currentPercentCompleted = (currentHasTier / currentTotalAssetCount) * 100;
     }
 
+    int changeCount = (int) (currentHasTier - previousHasTier);
+
     // TODO: Understand if we actually use this tierData for anything.
     Map<String, Double> tierData = new HashMap<>();
 
@@ -367,6 +386,7 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
         String.valueOf(currentHasTier.intValue()),
         currentPercentCompleted,
         KPI_NOT_SET,
+        changeCount,
         currentPercentCompleted - previousPercentCompleted,
         false,
         "",
@@ -444,6 +464,7 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
       DataInsightDescriptionAndOwnerTemplate.MetricType metricType,
       String chartKpiName,
       Double percentCompleted,
+      int changeCount,
       Double percentChange,
       int totalAssets,
       int numberOfDaysChange,
@@ -490,6 +511,7 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
         String.valueOf(totalAssets),
         percentCompleted,
         targetKpi,
+        changeCount,
         percentChange,
         isKpiAvailable,
         totalDaysLeft,

openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightDescriptionAndOwnerTemplate.java

@@ -33,6 +33,7 @@ public class DataInsightDescriptionAndOwnerTemplate {
   }
 
   @Setter private String totalAssets;
+  private final String changeCount;
   private final String percentCompleted;
   @Setter private boolean kpiAvailable;
   private String percentChange;
@@ -50,6 +51,7 @@ public class DataInsightDescriptionAndOwnerTemplate {
       String totalAssets,
       Double percentCompleted,
       String targetKpi,
+      int changeCount,
       Double percentChange,
       boolean isKpiAvailable,
       String numberOfDaysLeft,
@@ -58,6 +60,7 @@ public class DataInsightDescriptionAndOwnerTemplate {
       Map<String, Integer> dateMap) {
     this.percentCompleted = String.format("%.2f", percentCompleted);
     this.targetKpi = targetKpi;
+    this.changeCount = String.valueOf(changeCount);
     this.percentChange = String.format("%.2f", percentChange);
     this.percentChangeMessage = getFormattedPercentChangeMessage(percentChange);
     this.totalAssets = totalAssets;

openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightTotalAssetTemplate.java

@@ -23,6 +23,7 @@ import lombok.Setter;
 @SuppressWarnings("unused")
 public class DataInsightTotalAssetTemplate {
   private String totalDataAssets;
+  private final String changeInTotalAssets;
   private String percentChangeTotalAssets;
   @Setter private String percentChangeMessage;
   @Setter private String completeMessage;
@@ -31,10 +32,12 @@ public class DataInsightTotalAssetTemplate {
 
   public DataInsightTotalAssetTemplate(
       String totalDataAssets,
+      int assetsAddedOrRemoved,
       Double percentChangeTotalAssets,
       int numberOfDaysChange,
       Map<String, Integer> dateMap) {
     this.totalDataAssets = totalDataAssets;
+    this.changeInTotalAssets = String.valueOf(assetsAddedOrRemoved);
     this.percentChangeTotalAssets = String.format("%.2f", percentChangeTotalAssets);
     this.percentChangeMessage = getFormattedPercentChangeMessage(percentChangeTotalAssets);
     this.numberOfDaysChange = numberOfDaysChange;

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DocumentRepository.java

@@ -15,16 +15,21 @@ package org.openmetadata.service.jdbi3;
 
 import static org.openmetadata.service.Entity.DOCUMENT;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import lombok.extern.slf4j.Slf4j;
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.schema.email.EmailTemplate;
+import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.email.TemplateValidationResponse;
 import org.openmetadata.schema.entities.docStore.Data;
 import org.openmetadata.schema.entities.docStore.Document;
+import org.openmetadata.schema.settings.SettingsType;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.exception.EntityNotFoundException;
 import org.openmetadata.service.resources.docstore.DocStoreResource;
+import org.openmetadata.service.resources.settings.SettingsCache;
 import org.openmetadata.service.util.DefaultTemplateProvider;
 import org.openmetadata.service.util.EntityUtil.Fields;
 import org.openmetadata.service.util.JsonUtils;
@@ -36,6 +41,7 @@ public class DocumentRepository extends EntityRepository<Document> {
   static final String DOCUMENT_PATCH_FIELDS = "data";
   private final CollectionDAO.DocStoreDAO dao;
   private final TemplateProvider templateProvider;
+  private final String COLLATE = "collate";
 
   public DocumentRepository() {
     super(
@@ -50,6 +56,28 @@ public class DocumentRepository extends EntityRepository<Document> {
     this.templateProvider = new DefaultTemplateProvider();
   }
 
+  @Override
+  public List<Document> getEntitiesFromSeedData() throws IOException {
+    List<Document> entitiesFromSeedData = new ArrayList<>();
+    SmtpSettings emailConfig =
+        SettingsCache.getSetting(SettingsType.EMAIL_CONFIGURATION, SmtpSettings.class);
+
+    if (emailConfig.getTemplates().value().equals(COLLATE)) {
+      entitiesFromSeedData.addAll(
+          getEntitiesFromSeedData(
+              String.format(".*json/data/%s/emailTemplates/collate/.*\\.json$", entityType)));
+    } else {
+      entitiesFromSeedData.addAll(
+          getEntitiesFromSeedData(
+              String.format(".*json/data/%s/emailTemplates/openmetadata/.*\\.json$", entityType)));
+    }
+
+    entitiesFromSeedData.addAll(
+        getEntitiesFromSeedData(String.format(".*json/data/%s/docs/.*\\.json$", entityType)));
+
+    return entitiesFromSeedData;
+  }
+
   public List<Document> fetchAllEmailTemplates() {
     List<String> jsons = dao.fetchAllEmailTemplates();
     return jsons.stream().map(json -> JsonUtils.readValue(json, Document.class)).toList();

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java

@@ -117,12 +117,10 @@ import org.openmetadata.schema.api.VoteRequest;
 import org.openmetadata.schema.api.VoteRequest.VoteType;
 import org.openmetadata.schema.api.feed.ResolveTask;
 import org.openmetadata.schema.api.teams.CreateTeam;
-import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.entity.data.Table;
 import org.openmetadata.schema.entity.feed.Suggestion;
 import org.openmetadata.schema.entity.teams.Team;
 import org.openmetadata.schema.entity.teams.User;
-import org.openmetadata.schema.settings.SettingsType;
 import org.openmetadata.schema.system.EntityError;
 import org.openmetadata.schema.type.ApiStatus;
 import org.openmetadata.schema.type.ChangeDescription;
@@ -157,7 +155,6 @@ import org.openmetadata.service.jdbi3.CollectionDAO.EntityVersionPair;
 import org.openmetadata.service.jdbi3.CollectionDAO.ExtensionRecord;
 import org.openmetadata.service.jdbi3.FeedRepository.TaskWorkflow;
 import org.openmetadata.service.jdbi3.FeedRepository.ThreadContext;
-import org.openmetadata.service.resources.settings.SettingsCache;
 import org.openmetadata.service.resources.tags.TagLabelUtil;
 import org.openmetadata.service.search.SearchClient;
 import org.openmetadata.service.search.SearchListFilter;
@@ -445,32 +442,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
     }
   }
 
-  public final List<T> getEntitiesFromSeedData() throws IOException {
+  public List<T> getEntitiesFromSeedData() throws IOException {
-    List<T> entitiesFromSeedData = new ArrayList<>();
-
-    if (entityType.equals(Entity.DOCUMENT)) {
-      SmtpSettings emailConfig =
-          SettingsCache.getSetting(SettingsType.EMAIL_CONFIGURATION, SmtpSettings.class);
-
-      switch (emailConfig.getTemplates()) {
-        case COLLATE -> {
-          entitiesFromSeedData.addAll(
-              getEntitiesFromSeedData(
-                  String.format(".*json/data/%s/emailTemplates/collate/.*\\.json$", entityType)));
-        }
-        default -> {
-          entitiesFromSeedData.addAll(
-              getEntitiesFromSeedData(
-                  String.format(
-                      ".*json/data/%s/emailTemplates/openmetadata/.*\\.json$", entityType)));
-        }
-      }
-
-      entitiesFromSeedData.addAll(
-          getEntitiesFromSeedData(String.format(".*json/data/%s/docs/.*\\.json$", entityType)));
-      return entitiesFromSeedData;
-    }
-
     return getEntitiesFromSeedData(String.format(".*json/data/%s/.*\\.json$", entityType));
   }
 

openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v153/Migration.java

@@ -0,0 +1,20 @@
+package org.openmetadata.service.migration.mysql.v153;
+
+import static org.openmetadata.service.migration.utils.v153.MigrationUtil.updateEmailTemplates;
+
+import lombok.SneakyThrows;
+import org.openmetadata.service.migration.api.MigrationProcessImpl;
+import org.openmetadata.service.migration.utils.MigrationFile;
+
+public class Migration extends MigrationProcessImpl {
+
+  public Migration(MigrationFile migrationFile) {
+    super(migrationFile);
+  }
+
+  @Override
+  @SneakyThrows
+  public void runDataMigration() {
+    updateEmailTemplates(collectionDAO);
+  }
+}

openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v153/Migration.java

@@ -0,0 +1,20 @@
+package org.openmetadata.service.migration.postgres.v153;
+
+import static org.openmetadata.service.migration.utils.v153.MigrationUtil.updateEmailTemplates;
+
+import lombok.SneakyThrows;
+import org.openmetadata.service.migration.api.MigrationProcessImpl;
+import org.openmetadata.service.migration.utils.MigrationFile;
+
+public class Migration extends MigrationProcessImpl {
+
+  public Migration(MigrationFile migrationFile) {
+    super(migrationFile);
+  }
+
+  @Override
+  @SneakyThrows
+  public void runDataMigration() {
+    updateEmailTemplates(collectionDAO);
+  }
+}

openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v153/MigrationUtil.java

@@ -0,0 +1,14 @@
+package org.openmetadata.service.migration.utils.v153;
+
+import lombok.extern.slf4j.Slf4j;
+import org.openmetadata.service.jdbi3.CollectionDAO;
+
+@Slf4j
+public class MigrationUtil {
+
+  public static void updateEmailTemplates(CollectionDAO collectionDAO) {
+    CollectionDAO.DocStoreDAO dao = collectionDAO.docStoreDAO();
+    // delete emailTemplates, it will be loaded from initSeedData.
+    dao.deleteEmailTemplates();
+  }
+}

openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java

@@ -108,7 +108,6 @@ import org.openmetadata.schema.auth.SSOAuthMechanism;
 import org.openmetadata.schema.auth.ServiceTokenType;
 import org.openmetadata.schema.auth.TokenRefreshRequest;
 import org.openmetadata.schema.auth.TokenType;
-import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.services.connections.metadata.AuthProvider;
@@ -173,7 +172,6 @@ public class UserResource extends EntityResource<User, UserRepository> {
   public static final String USER_PROTECTED_FIELDS = "authenticationMechanism";
   private final JWTTokenGenerator jwtTokenGenerator;
   private final TokenRepository tokenRepository;
-  private boolean isEmailServiceEnabled;
   private AuthenticationConfiguration authenticationConfiguration;
   private AuthorizerConfiguration authorizerConfiguration;
   private final AuthenticatorHandler authHandler;
@@ -192,10 +190,6 @@ public class UserResource extends EntityResource<User, UserRepository> {
     return user;
   }
 
-  private boolean isEmailServiceEnabled() {
-    return getSmtpSettings().getEnableSmtpServer();
-  }
-
   public UserResource(
       Authorizer authorizer, Limits limits, AuthenticatorHandler authenticatorHandler) {
     super(Entity.USER, authorizer, limits);
@@ -217,8 +211,6 @@ public class UserResource extends EntityResource<User, UserRepository> {
     super.initialize(config);
     this.authenticationConfiguration = config.getAuthenticationConfiguration();
     this.authorizerConfiguration = config.getAuthorizerConfiguration();
-    SmtpSettings smtpSettings = config.getSmtpSettings();
-    this.isEmailServiceEnabled = smtpSettings != null && smtpSettings.getEnableSmtpServer();
     this.repository.initializeUsers(config);
     this.isSelfSignUpEnabled = authenticationConfiguration.getEnableSelfSignup();
   }
@@ -647,7 +639,7 @@ public class UserResource extends EntityResource<User, UserRepository> {
   }
 
   private void sendInviteMailToUserForBasicAuth(UriInfo uriInfo, User user, CreateUser create) {
-    if (isBasicAuth() && isEmailServiceEnabled()) {
+    if (isBasicAuth() && getSmtpSettings().getEnableSmtpServer()) {
       try {
         authHandler.sendInviteMailToUser(
             uriInfo,

openmetadata-service/src/main/java/org/openmetadata/service/security/auth/BasicAuthenticator.java

@@ -70,7 +70,6 @@ import org.openmetadata.schema.auth.RefreshToken;
 import org.openmetadata.schema.auth.RegistrationRequest;
 import org.openmetadata.schema.auth.ServiceTokenType;
 import org.openmetadata.schema.auth.TokenRefreshRequest;
-import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.service.Entity;
@@ -97,7 +96,6 @@ public class BasicAuthenticator implements AuthenticatorHandler {
   private TokenRepository tokenRepository;
   private LoginAttemptCache loginAttemptCache;
   private AuthorizerConfiguration authorizerConfiguration;
-  private boolean isEmailServiceEnabled;
   private boolean isSelfSignUpAvailable;
 
   @Override
@@ -106,15 +104,9 @@ public class BasicAuthenticator implements AuthenticatorHandler {
     this.tokenRepository = Entity.getTokenRepository();
     this.authorizerConfiguration = config.getAuthorizerConfiguration();
     this.loginAttemptCache = new LoginAttemptCache();
-    SmtpSettings smtpSettings = config.getSmtpSettings();
-    this.isEmailServiceEnabled = smtpSettings != null && smtpSettings.getEnableSmtpServer();
     this.isSelfSignUpAvailable = config.getAuthenticationConfiguration().getEnableSelfSignup();
   }
 
-  private boolean isEmailServiceEnabled() {
-    return getSmtpSettings().getEnableSmtpServer();
-  }
-
   @Override
   public User registerUser(RegistrationRequest newRegistrationRequest) {
     if (isSelfSignUpAvailable) {
@@ -177,7 +169,7 @@ public class BasicAuthenticator implements AuthenticatorHandler {
 
   @Override
   public void sendEmailVerification(UriInfo uriInfo, User user) throws IOException {
-    if (isEmailServiceEnabled()) {
+    if (getSmtpSettings().getEnableSmtpServer()) {
       UUID mailVerificationToken = UUID.randomUUID();
       EmailVerificationToken emailVerificationToken =
           TokenUtil.getEmailVerificationToken(user.getId(), mailVerificationToken);
@@ -314,7 +306,7 @@ public class BasicAuthenticator implements AuthenticatorHandler {
     loginAttemptCache.recordSuccessfulLogin(userName);
 
     // in case admin updates , send email to user
-    if (request.getRequestType() == USER && isEmailServiceEnabled()) {
+    if (request.getRequestType() == USER && getSmtpSettings().getEnableSmtpServer()) {
       // Send mail
       sendInviteMailToUser(
           uriInfo,