Data insights report app update (#17235)

openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java

@@ -1,18 +1,12 @@
 package org.openmetadata.service.apps.bundles.insights;
 
-import static org.openmetadata.schema.dataInsight.DataInsightChartResult.DataInsightChartType.PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE;
-import static org.openmetadata.schema.dataInsight.DataInsightChartResult.DataInsightChartType.PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE;
-import static org.openmetadata.schema.dataInsight.DataInsightChartResult.DataInsightChartType.TOTAL_ENTITIES_BY_TIER;
-import static org.openmetadata.schema.dataInsight.DataInsightChartResult.DataInsightChartType.TOTAL_ENTITIES_BY_TYPE;
 import static org.openmetadata.schema.entity.events.SubscriptionDestination.SubscriptionType.EMAIL;
-import static org.openmetadata.schema.type.DataReportIndex.ENTITY_REPORT_DATA_INDEX;
 import static org.openmetadata.service.Entity.KPI;
 import static org.openmetadata.service.Entity.TEAM;
 import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_NAME;
 import static org.openmetadata.service.util.SubscriptionUtil.getAdminsData;
 import static org.openmetadata.service.util.Utilities.getMonthAndDateFromEpoch;
 
-import com.fasterxml.jackson.core.type.TypeReference;
 import java.io.IOException;
 import java.text.ParseException;
 import java.time.Instant;
@@ -21,20 +15,15 @@ import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.TreeMap;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.Pair;
 import org.openmetadata.common.utils.CommonUtil;
-import org.openmetadata.schema.dataInsight.DataInsightChartResult;
+import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChartResultList;
 import org.openmetadata.schema.dataInsight.kpi.Kpi;
 import org.openmetadata.schema.dataInsight.type.KpiResult;
-import org.openmetadata.schema.dataInsight.type.PercentageOfEntitiesWithDescriptionByType;
-import org.openmetadata.schema.dataInsight.type.PercentageOfEntitiesWithOwnerByType;
-import org.openmetadata.schema.dataInsight.type.TotalEntitiesByTier;
-import org.openmetadata.schema.dataInsight.type.TotalEntitiesByType;
 import org.openmetadata.schema.entity.app.App;
 import org.openmetadata.schema.entity.applications.configuration.internal.DataInsightsReportAppConfig;
 import org.openmetadata.schema.entity.teams.Team;
@@ -43,11 +32,13 @@ import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.apps.AbstractNativeApplication;
+import org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils;
 import org.openmetadata.service.events.scheduled.template.DataInsightDescriptionAndOwnerTemplate;
 import org.openmetadata.service.events.scheduled.template.DataInsightTotalAssetTemplate;
 import org.openmetadata.service.exception.EventSubscriptionJobException;
 import org.openmetadata.service.exception.SearchIndexException;
 import org.openmetadata.service.jdbi3.CollectionDAO;
+import org.openmetadata.service.jdbi3.DataInsightSystemChartRepository;
 import org.openmetadata.service.jdbi3.KpiRepository;
 import org.openmetadata.service.jdbi3.ListFilter;
 import org.openmetadata.service.search.SearchClient;
@@ -63,6 +54,13 @@ import org.quartz.JobExecutionContext;
 @SuppressWarnings("unused")
 public class DataInsightsReportApp extends AbstractNativeApplication {
   private static final String KPI_NOT_SET = "No Kpi Set";
+  private static final String PREVIOUS_TOTAL_ASSET_COUNT = "PreviousTotalAssetCount";
+  private static final String CURRENT_TOTAL_ASSET_COUNT = "CurrentTotalAssetCount";
+  private final DataInsightSystemChartRepository systemChartRepository =
+      new DataInsightSystemChartRepository();
+
+  private record TimeConfig(
+      Long startTime, Long endTime, String startDay, String endDay, int numberOfDaysChange) {}
 
   public DataInsightsReportApp(CollectionDAO collectionDAO, SearchRepository searchRepository) {
     super(collectionDAO, searchRepository);
@@ -72,23 +70,30 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
   public void execute(JobExecutionContext jobExecutionContext) {
     String appName = (String) jobExecutionContext.getJobDetail().getJobDataMap().get(APP_NAME);
     App app = collectionDAO.applicationDAO().findEntityByName(appName);
-    // Calculate time diff
+
+    // Calculate time config
     long currentTime = Instant.now().toEpochMilli();
-    long scheduleTime = currentTime - 604800000L;
+    long startTime = TimestampUtils.subtractDays(currentTime, 7);
-    int numberOfDaysChange = 7;
+    long endTime = TimestampUtils.subtractDays(currentTime, 1);
+    TimeConfig timeConfig =
+        new TimeConfig(
+            startTime,
+            endTime,
+            TimestampUtils.timestampToString(startTime, "dd"),
+            TimestampUtils.timestampToString(endTime, "dd"),
+            7);
+
     try {
       DataInsightsReportAppConfig insightAlertConfig =
           JsonUtils.convertValue(app.getAppConfiguration(), DataInsightsReportAppConfig.class);
       // Send to Admins
       if (Boolean.TRUE.equals(insightAlertConfig.getSendToAdmins())) {
-        sendToAdmins(
+        sendToAdmins(searchRepository.getSearchClient(), timeConfig);
-            searchRepository.getSearchClient(), scheduleTime, currentTime, numberOfDaysChange);
       }
 
       // Send to Teams
       if (Boolean.FALSE.equals(insightAlertConfig.getSendToTeams())) {
-        sendReportsToTeams(
+        sendReportsToTeams(searchRepository.getSearchClient(), timeConfig);
-            searchRepository.getSearchClient(), scheduleTime, currentTime, numberOfDaysChange);
       }
     } catch (Exception e) {
       LOG.error("[DIReport] Failed in sending report due to", e);
@@ -96,8 +101,7 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
     }
   }
 
-  private void sendReportsToTeams(
+  private void sendReportsToTeams(SearchClient searchClient, TimeConfig timeConfig)
-      SearchClient searchClient, Long scheduleTime, Long currentTime, int numberOfDaysChange)
       throws SearchIndexException {
     PaginatedEntitiesSource teamReader =
         new PaginatedEntitiesSource(TEAM, 10, List.of("name", "email", "users"));
@@ -115,23 +119,21 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
           }
         }
 
+        Map<String, Object> contextData = new HashMap<>();
+
         try {
           DataInsightTotalAssetTemplate totalAssetTemplate =
-              createTotalAssetTemplate(
+              createTotalAssetTemplate(searchClient, team.getName(), timeConfig, contextData);
-                  searchClient, team.getName(), scheduleTime, currentTime, numberOfDaysChange);
           DataInsightDescriptionAndOwnerTemplate descriptionTemplate =
-              createDescriptionTemplate(
+              createDescriptionTemplate(searchClient, team.getName(), timeConfig, contextData);
-                  searchClient, team.getName(), scheduleTime, currentTime, numberOfDaysChange);
           DataInsightDescriptionAndOwnerTemplate ownershipTemplate =
-              createOwnershipTemplate(
+              createOwnershipTemplate(searchClient, team.getName(), timeConfig, contextData);
-                  searchClient, team.getName(), scheduleTime, currentTime, numberOfDaysChange);
           DataInsightDescriptionAndOwnerTemplate tierTemplate =
-              createTierTemplate(
+              createTierTemplate(searchClient, team.getName(), timeConfig, contextData);
-                  searchClient, team.getName(), scheduleTime, currentTime, numberOfDaysChange);
           EmailUtil.sendDataInsightEmailNotificationToUser(
               emails,
-              getMonthAndDateFromEpoch(scheduleTime),
+              getMonthAndDateFromEpoch(timeConfig.startTime()),
-              getMonthAndDateFromEpoch(currentTime),
+              getMonthAndDateFromEpoch(timeConfig.endTime()),
               totalAssetTemplate,
               descriptionTemplate,
               ownershipTemplate,
@@ -148,27 +150,25 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
     }
   }
 
-  private void sendToAdmins(
+  private void sendToAdmins(SearchClient searchClient, TimeConfig timeConfig) {
-      SearchClient searchClient, Long scheduleTime, Long currentTime, int numberOfDaysChange) {
     // Get Admins
     Set<String> emailList = getAdminsData(EMAIL);
+    Map<String, Object> contextData = new HashMap<>();
+
     try {
       // Build Insights Report
       DataInsightTotalAssetTemplate totalAssetTemplate =
-          createTotalAssetTemplate(
+          createTotalAssetTemplate(searchClient, null, timeConfig, contextData);
-              searchClient, null, scheduleTime, currentTime, numberOfDaysChange);
       DataInsightDescriptionAndOwnerTemplate descriptionTemplate =
-          createDescriptionTemplate(
+          createDescriptionTemplate(searchClient, null, timeConfig, contextData);
-              searchClient, null, scheduleTime, currentTime, numberOfDaysChange);
       DataInsightDescriptionAndOwnerTemplate ownershipTemplate =
-          createOwnershipTemplate(
+          createOwnershipTemplate(searchClient, null, timeConfig, contextData);
-              searchClient, null, scheduleTime, currentTime, numberOfDaysChange);
       DataInsightDescriptionAndOwnerTemplate tierTemplate =
-          createTierTemplate(searchClient, null, scheduleTime, currentTime, numberOfDaysChange);
+          createTierTemplate(searchClient, null, timeConfig, contextData);
       EmailUtil.sendDataInsightEmailNotificationToUser(
           emailList,
-          getMonthAndDateFromEpoch(scheduleTime),
+          getMonthAndDateFromEpoch(timeConfig.startTime()),
-          getMonthAndDateFromEpoch(currentTime),
+          getMonthAndDateFromEpoch(timeConfig.endTime()),
           totalAssetTemplate,
           descriptionTemplate,
           ownershipTemplate,
@@ -192,321 +192,220 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
   }
 
   private DataInsightTotalAssetTemplate createTotalAssetTemplate(
-      SearchClient searchClient, String team, Long scheduleTime, Long currentTime, int numberOfDays)
+      SearchClient searchClient,
-      throws ParseException, IOException {
+      String team,
+      TimeConfig timeConfig,
+      Map<String, Object> contextData)
+      throws IOException {
     // Create A Date Map
     Map<String, Integer> dateMap = new LinkedHashMap<>();
-    Utilities.getLastSevenDays(currentTime).forEach(day -> dateMap.put(day, 0));
+    Utilities.getLastSevenDays(timeConfig.endTime()).forEach(day -> dateMap.put(day, 0));
     // Get total Assets Data
-    TreeMap<Long, List<Object>> dateWithDataMap =
+    Map<String, Double> dateWithCount =
-        searchClient.getSortedDate(
+        getDateMapWithCountFromChart(
-            team,
+            "total_data_assets", timeConfig.startTime(), timeConfig.endTime());
-            scheduleTime,
-            currentTime,
-            TOTAL_ENTITIES_BY_TYPE,
-            ENTITY_REPORT_DATA_INDEX.value());
-    if (dateWithDataMap.firstEntry() != null && dateWithDataMap.lastEntry() != null) {
-      List<TotalEntitiesByType> first =
-          JsonUtils.convertValue(dateWithDataMap.firstEntry().getValue(), new TypeReference<>() {});
-      List<TotalEntitiesByType> last =
-          JsonUtils.convertValue(dateWithDataMap.lastEntry().getValue(), new TypeReference<>() {});
-      Double previousCount = getCountOfEntitiesFromList(first);
-      Double currentCount = getCountOfEntitiesFromList(last);
 
-      dateWithDataMap.forEach(
+    Double previousCount = dateWithCount.getOrDefault(timeConfig.startDay(), 0D);
-          (key, value) -> {
+    Double currentCount = dateWithCount.getOrDefault(timeConfig.endDay(), 0D);
-            List<TotalEntitiesByType> list =
-                JsonUtils.convertValue(value, new TypeReference<>() {});
-            Double count = getCountOfEntitiesFromList(list);
-            dateMap.put(Utilities.getDateFromEpoch(key), count.intValue());
-          });
 
-      processDateMapToNormalize(dateMap);
+    contextData.put(PREVIOUS_TOTAL_ASSET_COUNT, previousCount);
+    contextData.put(CURRENT_TOTAL_ASSET_COUNT, currentCount);
 
-      if (previousCount == 0D) {
+    dateWithCount.forEach(
-        // it should be undefined
+        (key, value) -> {
-        return new DataInsightTotalAssetTemplate(currentCount, 0D, numberOfDays, dateMap);
+          dateMap.put(key, value.intValue());
-      } else {
+        });
-        return new DataInsightTotalAssetTemplate(
+    processDateMapToNormalize(dateMap);
-            currentCount,
+
-            ((currentCount - previousCount) / previousCount) * 100,
+    if (previousCount == 0D) {
-            numberOfDays,
+      // it should be undefined
-            dateMap);
+      return new DataInsightTotalAssetTemplate(
-      }
+          currentCount, 0D, timeConfig.numberOfDaysChange(), dateMap);
+    } else {
+      return new DataInsightTotalAssetTemplate(
+          currentCount,
+          ((currentCount - previousCount) / previousCount) * 100,
+          timeConfig.numberOfDaysChange(),
+          dateMap);
     }
-
-    return new DataInsightTotalAssetTemplate(0D, 0D, numberOfDays, dateMap);
   }
 
   private DataInsightDescriptionAndOwnerTemplate createDescriptionTemplate(
       SearchClient searchClient,
       String team,
-      Long scheduleTime,
+      TimeConfig timeConfig,
-      Long currentTime,
+      Map<String, Object> contextData)
-      int numberOfDaysChange)
       throws ParseException, IOException {
     // Create A Date Map
     Map<String, Integer> dateMap = new LinkedHashMap<>();
-    Utilities.getLastSevenDays(currentTime).forEach(day -> dateMap.put(day, 0));
+    Utilities.getLastSevenDays(timeConfig.endTime()).forEach(day -> dateMap.put(day, 0));
     // Get total Assets Data
     // This assumes that on a particular date the correct count per entities are given
-    TreeMap<Long, List<Object>> dateWithDataMap =
+    Map<String, Double> dateWithCount =
-        searchClient.getSortedDate(
+        getDateMapWithCountFromChart(
-            team,
+            "number_of_data_asset_with_description_kpi",
-            scheduleTime,
+            timeConfig.startTime(),
-            currentTime,
+            timeConfig.endTime());
-            PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE,
-            ENTITY_REPORT_DATA_INDEX.value());
-    if (dateWithDataMap.firstEntry() != null && dateWithDataMap.lastEntry() != null) {
-      List<PercentageOfEntitiesWithDescriptionByType> first =
-          JsonUtils.convertValue(dateWithDataMap.firstEntry().getValue(), new TypeReference<>() {});
-      List<PercentageOfEntitiesWithDescriptionByType> last =
-          JsonUtils.convertValue(dateWithDataMap.lastEntry().getValue(), new TypeReference<>() {});
 
-      double previousCompletedDescription = getCompletedDescriptionCount(first);
+    Double previousCompletedDescription = dateWithCount.getOrDefault(timeConfig.startDay(), 0D);
-      double previousTotalCount = getTotalEntityFromDescriptionList(first);
+    Double currentCompletedDescription = dateWithCount.getOrDefault(timeConfig.endDay(), 0D);
-      double currentCompletedDescription = getCompletedDescriptionCount(last);
-      double currentTotalCount = getTotalEntityFromDescriptionList(last);
 
-      dateWithDataMap.forEach(
+    Double previousTotalAssetCount = (double) contextData.get(PREVIOUS_TOTAL_ASSET_COUNT);
-          (key, value) -> {
+    Double currentTotalAssetCount = (double) contextData.get(CURRENT_TOTAL_ASSET_COUNT);
-            List<PercentageOfEntitiesWithDescriptionByType> list =
-                JsonUtils.convertValue(value, new TypeReference<>() {});
-            Double count = getCompletedDescriptionCount(list);
-            dateMap.put(Utilities.getDateFromEpoch(key), count.intValue());
-          });
 
-      processDateMapToNormalize(dateMap);
+    dateWithCount.forEach(
+        (key, value) -> {
+          dateMap.put(key, value.intValue());
+        });
+    processDateMapToNormalize(dateMap);
 
-      // Previous Percent
+    // Previous Percent
-      double previousPercentCompleted = 0D;
+    double previousPercentCompleted = 0D;
-      if (previousTotalCount != 0) {
+    if (previousTotalAssetCount != 0D) {
-        previousPercentCompleted = (previousCompletedDescription / previousTotalCount) * 100;
+      previousPercentCompleted = (previousCompletedDescription / previousTotalAssetCount) * 100;
-      }
+    }
-      // Current Percent
+    // Current Percent
-      double currentPercentCompleted = 0;
+    double currentPercentCompleted = 0D;
-      if (currentTotalCount != 0) {
+    if (currentTotalAssetCount != 0D) {
-        currentPercentCompleted = (currentCompletedDescription / currentTotalCount) * 100;
+      currentPercentCompleted = (currentCompletedDescription / currentTotalAssetCount) * 100;
-      }
-
-      return getTemplate(
-          DataInsightDescriptionAndOwnerTemplate.MetricType.DESCRIPTION,
-          PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE,
-          currentPercentCompleted,
-          currentPercentCompleted - previousPercentCompleted,
-          (int) currentCompletedDescription,
-          numberOfDaysChange,
-          dateMap);
     }
 
     return getTemplate(
         DataInsightDescriptionAndOwnerTemplate.MetricType.DESCRIPTION,
-        PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE,
+        "percentage_of_data_asset_with_description_kpi",
-        0D,
+        currentPercentCompleted,
-        0D,
+        currentPercentCompleted - previousPercentCompleted,
-        0,
+        currentCompletedDescription.intValue(),
-        numberOfDaysChange,
+        timeConfig.numberOfDaysChange(),
         dateMap);
   }
 
   private DataInsightDescriptionAndOwnerTemplate createOwnershipTemplate(
       SearchClient searchClient,
       String team,
-      Long scheduleTime,
+      TimeConfig timeConfig,
-      Long currentTime,
+      Map<String, Object> contextData)
-      int numberOfDaysChange)
+      throws IOException {
-      throws ParseException, IOException {
     // Create A Date Map
     Map<String, Integer> dateMap = new LinkedHashMap<>();
-    Utilities.getLastSevenDays(currentTime).forEach(day -> dateMap.put(day, 0));
+    Utilities.getLastSevenDays(timeConfig.endTime()).forEach(day -> dateMap.put(day, 0));
     // Get total Assets Data
     // This assumes that on a particular date the correct count per entities are given
-    TreeMap<Long, List<Object>> dateWithDataMap =
+    Map<String, Double> dateWithCount =
-        searchClient.getSortedDate(
+        getDateMapWithCountFromChart(
-            team,
+            "number_of_data_asset_with_owner_kpi", timeConfig.startTime(), timeConfig.endTime());
-            scheduleTime,
-            currentTime,
-            PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE,
-            ENTITY_REPORT_DATA_INDEX.value());
-    if (dateWithDataMap.firstEntry() != null && dateWithDataMap.lastEntry() != null) {
-      List<PercentageOfEntitiesWithOwnerByType> first =
-          JsonUtils.convertValue(dateWithDataMap.firstEntry().getValue(), new TypeReference<>() {});
-      List<PercentageOfEntitiesWithOwnerByType> last =
-          JsonUtils.convertValue(dateWithDataMap.lastEntry().getValue(), new TypeReference<>() {});
 
-      double previousHasOwner = getCompletedOwnershipCount(first);
+    Double previousHasOwner = dateWithCount.getOrDefault(timeConfig.startDay(), 0D);
-      double previousTotalCount = getTotalEntityFromOwnerList(first);
+    Double currentHasOwner = dateWithCount.getOrDefault(timeConfig.endDay(), 0D);
-      double currentHasOwner = getCompletedOwnershipCount(last);
-      double currentTotalCount = getTotalEntityFromOwnerList(last);
 
-      // Previous Percent
+    Double previousTotalAssetCount = (double) contextData.get(PREVIOUS_TOTAL_ASSET_COUNT);
-      double previousPercentCompleted = 0D;
+    Double currentTotalAssetCount = (double) contextData.get(CURRENT_TOTAL_ASSET_COUNT);
-      if (previousTotalCount != 0) {
-        previousPercentCompleted = (previousHasOwner / previousTotalCount) * 100;
-      }
-      // Current Percent
-      double currentPercentCompleted = 0;
-      if (currentTotalCount != 0) {
-        currentPercentCompleted = (currentHasOwner / currentTotalCount) * 100;
-      }
-      dateWithDataMap.forEach(
-          (key, value) -> {
-            List<PercentageOfEntitiesWithOwnerByType> list =
-                JsonUtils.convertValue(value, new TypeReference<>() {});
-            Double count = getCompletedOwnershipCount(list);
-            dateMap.put(Utilities.getDateFromEpoch(key), count.intValue());
-          });
 
-      processDateMapToNormalize(dateMap);
+    dateWithCount.forEach(
+        (key, value) -> {
+          dateMap.put(key, value.intValue());
+        });
+    processDateMapToNormalize(dateMap);
 
-      return getTemplate(
+    // Previous Percent
-          DataInsightDescriptionAndOwnerTemplate.MetricType.OWNER,
+    double previousPercentCompleted = 0D;
-          PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE,
+    if (previousTotalAssetCount != 0) {
-          currentPercentCompleted,
+      previousPercentCompleted = (previousHasOwner / previousTotalAssetCount) * 100;
-          currentPercentCompleted - previousPercentCompleted,
-          (int) currentHasOwner,
-          numberOfDaysChange,
-          dateMap);
     }
+    // Current Percent
+    double currentPercentCompleted = 0;
+    if (currentTotalAssetCount != 0) {
+      currentPercentCompleted = (currentHasOwner / currentTotalAssetCount) * 100;
+    }
+
     return getTemplate(
         DataInsightDescriptionAndOwnerTemplate.MetricType.OWNER,
-        PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE,
+        "percentage_of_data_asset_with_owner_kpi",
-        0D,
+        currentPercentCompleted,
-        0D,
+        currentPercentCompleted - previousPercentCompleted,
-        0,
+        currentHasOwner.intValue(),
-        numberOfDaysChange,
+        timeConfig.numberOfDaysChange(),
         dateMap);
   }
 
   private DataInsightDescriptionAndOwnerTemplate createTierTemplate(
       SearchClient searchClient,
       String team,
-      Long scheduleTime,
+      TimeConfig timeConfig,
-      Long currentTime,
+      Map<String, Object> contextData)
-      int numberOfDaysChange)
       throws ParseException, IOException {
     // Create A Date Map
     Map<String, Integer> dateMap = new LinkedHashMap<>();
-    Utilities.getLastSevenDays(currentTime).forEach(day -> dateMap.put(day, 0));
+    Utilities.getLastSevenDays(timeConfig.endTime()).forEach(day -> dateMap.put(day, 0));
+
     // Get total Assets Data
     // This assumes that on a particular date the correct count per entities are given
-    TreeMap<Long, List<Object>> dateWithDataMap =
+    Map<String, Double> dateWithCount =
-        searchClient.getSortedDate(
+        getDateMapWithCountFromChart(
-            team,
+            "total_data_assets_by_tier", timeConfig.startTime(), timeConfig.endTime());
-            scheduleTime,
+
-            currentTime,
+    Double previousHasTier = dateWithCount.getOrDefault(timeConfig.startDay(), 0D);
-            TOTAL_ENTITIES_BY_TIER,
+    Double currentHasTier = dateWithCount.getOrDefault(timeConfig.endDay(), 0D);
-            ENTITY_REPORT_DATA_INDEX.value());
+
-    if (dateWithDataMap.lastEntry() != null) {
+    Double previousTotalAssetCount = (double) contextData.get(PREVIOUS_TOTAL_ASSET_COUNT);
-      List<TotalEntitiesByTier> last =
+    Double currentTotalAssetCount = (double) contextData.get(CURRENT_TOTAL_ASSET_COUNT);
-          JsonUtils.convertValue(dateWithDataMap.lastEntry().getValue(), new TypeReference<>() {});
+
-      dateWithDataMap.forEach(
+    dateWithCount.forEach(
-          (key, value) -> {
+        (key, value) -> {
-            List<TotalEntitiesByTier> list =
+          dateMap.put(key, value.intValue());
-                JsonUtils.convertValue(value, new TypeReference<>() {});
+        });
-            Double count = getCountOfTieredEntities(list);
+    processDateMapToNormalize(dateMap);
-            dateMap.put(Utilities.getDateFromEpoch(key), count.intValue());
+
-          });
+    // Previous Percent
-      processDateMapToNormalize(dateMap);
+    double previousPercentCompleted = 0D;
-      Map<String, Double> tierData = getTierData(last);
+    if (previousTotalAssetCount != 0) {
-      return new DataInsightDescriptionAndOwnerTemplate(
+      previousPercentCompleted = (previousHasTier / previousTotalAssetCount) * 100;
-          DataInsightDescriptionAndOwnerTemplate.MetricType.TIER,
-          null,
-          "0",
-          0D,
-          KPI_NOT_SET,
-          0D,
-          false,
-          "",
-          numberOfDaysChange,
-          tierData,
-          dateMap);
     }
+    // Current Percent
+    double currentPercentCompleted = 0;
+    if (currentTotalAssetCount != 0) {
+      currentPercentCompleted = (currentHasTier / currentTotalAssetCount) * 100;
+    }
+
+    // TODO: Understand if we actually use this tierData for anything.
+    Map<String, Double> tierData = new HashMap<>();
 
     return new DataInsightDescriptionAndOwnerTemplate(
         DataInsightDescriptionAndOwnerTemplate.MetricType.TIER,
         null,
-        "0",
+        String.valueOf(currentHasTier.intValue()),
-        0D,
+        currentPercentCompleted,
         KPI_NOT_SET,
-        0D,
+        currentPercentCompleted - previousPercentCompleted,
         false,
         "",
-        numberOfDaysChange,
+        timeConfig.numberOfDaysChange(),
-        new HashMap<>(),
+        tierData,
         dateMap);
   }
 
-  private Double getCountOfEntitiesFromList(List<TotalEntitiesByType> entitiesByTypeList) {
+  private Map<String, Double> getDateMapWithCountFromChart(
-    // If there are multiple entries for same entities then this can yield invalid results
+      String chartName, Long startTime, Long endTime) throws IOException {
-    Double totalCount = 0D;
+    Map<String, DataInsightCustomChartResultList> systemChartMap =
-    for (TotalEntitiesByType obj : entitiesByTypeList) {
+        systemChartRepository.listChartData(chartName, startTime, endTime);
-      totalCount += obj.getEntityCount();
+    return systemChartMap.get(chartName).getResults().stream()
-    }
+        .map(
-    return totalCount;
+            result -> {
-  }
+              Map<String, Double> dayCount = new HashMap<>();
-
+              dayCount.put(
-  private Double getCountOfTieredEntities(List<TotalEntitiesByTier> entitiesByTierList) {
+                  TimestampUtils.timestampToString(result.getDay().longValue(), "dd"),
-    // If there are multiple entries for same entities then this can yield invalid results
+                  result.getCount());
-    double totalCount = 0D;
+              return dayCount;
-    for (TotalEntitiesByTier obj : entitiesByTierList) {
+            })
-      totalCount += obj.getEntityCountFraction() * 100;
+        .flatMap(map -> map.entrySet().stream())
-    }
+        .collect(
-    return totalCount;
+            Collectors.groupingBy(
-  }
+                Map.Entry::getKey, Collectors.summingDouble(Map.Entry::getValue)));
-
-  private Map<String, Double> getTierData(List<TotalEntitiesByTier> entitiesByTypeList) {
-    // If there are multiple entries for same entities then this can yield invalid results
-    Map<String, Double> data = new TreeMap<>();
-    for (TotalEntitiesByTier obj : entitiesByTypeList) {
-      data.put(obj.getEntityTier(), obj.getEntityCountFraction() * 100);
-    }
-    return data;
-  }
-
-  private Double getTotalEntityFromDescriptionList(
-      List<PercentageOfEntitiesWithDescriptionByType> entitiesByTypeList) {
-    // If there are multiple entries for same entities then this can yield invalid results
-    Double totalCount = 0D;
-    for (PercentageOfEntitiesWithDescriptionByType obj : entitiesByTypeList) {
-      totalCount += obj.getEntityCount();
-    }
-    return totalCount;
-  }
-
-  private Double getCompletedDescriptionCount(
-      List<PercentageOfEntitiesWithDescriptionByType> entitiesByTypeList) {
-    // If there are multiple entries for same entities then this can yield invalid results
-    Double completedDescriptions = 0D;
-    for (PercentageOfEntitiesWithDescriptionByType obj : entitiesByTypeList) {
-      completedDescriptions += obj.getCompletedDescription();
-    }
-    return completedDescriptions;
-  }
-
-  private Double getTotalEntityFromOwnerList(
-      List<PercentageOfEntitiesWithOwnerByType> entitiesByTypeList) {
-    // If there are multiple entries for same entities then this can yield invalid results
-    Double totalCount = 0D;
-    for (PercentageOfEntitiesWithOwnerByType obj : entitiesByTypeList) {
-      totalCount += obj.getEntityCount();
-    }
-    return totalCount;
-  }
-
-  private Double getCompletedOwnershipCount(
-      List<PercentageOfEntitiesWithOwnerByType> entitiesByTypeList) {
-    // If there are multiple entries for same entities then this can yield invalid results
-    Double hasOwner = 0D;
-    for (PercentageOfEntitiesWithOwnerByType obj : entitiesByTypeList) {
-      hasOwner += obj.getHasOwner();
-    }
-    return hasOwner;
   }
 
   private DataInsightDescriptionAndOwnerTemplate getTemplate(
       DataInsightDescriptionAndOwnerTemplate.MetricType metricType,
-      DataInsightChartResult.DataInsightChartType chartType,
+      String chartKpiName,
       Double percentCompleted,
       Double percentChange,
       int totalAssets,
@@ -518,7 +417,7 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
     Kpi validKpi = null;
     boolean isKpiAvailable = false;
     for (Kpi kpiObj : kpiList) {
-      if (Objects.equals(kpiObj.getDataInsightChart().getName(), chartType.value())) {
+      if (kpiObj.getDataInsightChart().getName().equals(chartKpiName)) {
         validKpi = kpiObj;
         isKpiAvailable = true;
         break;
@@ -531,7 +430,7 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
     String targetKpi = KPI_NOT_SET;
 
     if (isKpiAvailable) {
-      targetKpi = String.format("%.2f", validKpi.getTargetValue() * 100);
+      targetKpi = String.format("%.2f", validKpi.getTargetValue());
       KpiResult result = getKpiResult(validKpi.getName());
       if (result != null) {
         isTargetMet = result.getTargetResult().get(0).getTargetMet();

openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/utils/TimestampUtils.java

@@ -32,7 +32,7 @@ public class TimestampUtils {
     String inputString = Instant.ofEpochMilli(timestamp).toString();
     DateTimeFormatter inputFormatter = DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.of("UTC"));
     ZonedDateTime zonedDateTime = ZonedDateTime.parse(inputString, inputFormatter);
-    DateTimeFormatter outputFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
+    DateTimeFormatter outputFormatter = DateTimeFormatter.ofPattern(pattern);
     return zonedDateTime.format(outputFormatter);
   }
 

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightSystemChartRepository.java

@@ -4,6 +4,7 @@ import static org.openmetadata.service.Entity.DATA_INSIGHT_CUSTOM_CHART;
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.Map;
 import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart;
 import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChartResultList;
 import org.openmetadata.schema.type.Include;
@@ -63,8 +64,8 @@ public class DataInsightSystemChartRepository extends EntityRepository<DataInsig
     return searchClient.buildDIChart(chart, startTimestamp, endTimestamp);
   }
 
-  public HashMap listChartData(String chartNames, long startTimestamp, long endTimestamp)
+  public Map<String, DataInsightCustomChartResultList> listChartData(
-      throws IOException {
+      String chartNames, long startTimestamp, long endTimestamp) throws IOException {
     HashMap<String, DataInsightCustomChartResultList> result = new HashMap<>();
     if (chartNames == null) {
       return result;

openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/system/DataInsightSystemChartResource.java

@@ -8,8 +8,8 @@ import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import javax.ws.rs.Consumes;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
@@ -140,7 +140,7 @@ public class DataInsightSystemChartResource
           @QueryParam("end")
           long end)
       throws IOException {
-    HashMap<String, DataInsightCustomChartResultList> resultList =
+    Map<String, DataInsightCustomChartResultList> resultList =
         repository.listChartData(chartNames, start, end);
     return Response.status(Response.Status.OK).entity(resultList).build();
   }