yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-27 16:55:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add new Template and Graph building logic for Insights reporting (#15184)

Browse Source
This commit is contained in:
Mohit Yadav 2024-02-15 13:45:45 +05:30 committed by GitHub
parent 9e867fb279
commit 0b7a4f8839
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 824 additions and 652 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

205
openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java
View File

@ -11,21 +11,23 @@ import static org.openmetadata.service.Entity.TEAM;
import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY;
import static org.openmetadata.service.apps.scheduler.AppScheduler.SEARCH_CLIENT_KEY;
import static org.openmetadata.service.util.SubscriptionUtil.getAdminsData;
import static org.openmetadata.service.util.Utilities.getMonthAndDateFromEpoch;
import com.fasterxml.jackson.core.type.TypeReference;
import java.io.IOException;
import java.text.ParseException;
import java.time.Instant;
import java.time.Period;
import java.time.ZoneId;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.dataInsight.kpi.Kpi;
@ -35,7 +37,6 @@ import org.openmetadata.schema.dataInsight.type.PercentageOfEntitiesWithOwnerByT
import org.openmetadata.schema.dataInsight.type.TotalEntitiesByTier;
import org.openmetadata.schema.dataInsight.type.TotalEntitiesByType;
import org.openmetadata.schema.entity.app.App;
import org.openmetadata.schema.entity.app.AppSchedule;
import org.openmetadata.schema.entity.applications.configuration.internal.DataInsightsReportAppConfig;
import org.openmetadata.schema.entity.teams.Team;
import org.openmetadata.schema.entity.teams.User;
@ -54,15 +55,13 @@ import org.openmetadata.service.search.SearchRepository;
import org.openmetadata.service.util.EmailUtil;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ResultList;
import org.openmetadata.service.util.Utilities;
import org.openmetadata.service.workflows.searchIndex.PaginatedEntitiesSource;
import org.quartz.CronScheduleBuilder;
import org.quartz.JobExecutionContext;
import org.quartz.Trigger;
@Slf4j
@SuppressWarnings("unused")
public class DataInsightsReportApp extends AbstractNativeApplication {
private static final String MISSING_DATA =
"Data Insight Report Data Unavailable or too short of a span for Reporting.";
private static final String KPI_NOT_SET = "No Kpi Set";
@Override
@ -73,9 +72,8 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
App app = (App) jobExecutionContext.getJobDetail().getJobDataMap().get(APP_INFO_KEY);
// Calculate time diff
long currentTime = Instant.now().toEpochMilli();
AppSchedule scheduleConfiguration = app.getAppSchedule();
long scheduleTime = currentTime - 604800000L;
int numberOfDaysChange = getNumberOfDays(scheduleConfiguration);
int numberOfDaysChange = 7;
try {
DataInsightsReportAppConfig insightAlertConfig =
JsonUtils.convertValue(app.getAppConfiguration(), DataInsightsReportAppConfig.class);
@ -86,7 +84,7 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
}
// Send to Teams
if (Boolean.TRUE.equals(insightAlertConfig.getSendToTeams())) {
if (Boolean.FALSE.equals(insightAlertConfig.getSendToTeams())) {
sendReportsToTeams(
searchRepository.getSearchClient(), scheduleTime, currentTime, numberOfDaysChange);
}
@ -130,6 +128,8 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
searchClient, team.getName(), scheduleTime, currentTime, numberOfDaysChange);
EmailUtil.sendDataInsightEmailNotificationToUser(
emails,
getMonthAndDateFromEpoch(scheduleTime),
getMonthAndDateFromEpoch(currentTime),
totalAssetTemplate,
descriptionTemplate,
ownershipTemplate,
@ -165,6 +165,8 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
createTierTemplate(searchClient, null, scheduleTime, currentTime, numberOfDaysChange);
EmailUtil.sendDataInsightEmailNotificationToUser(
emailList,
getMonthAndDateFromEpoch(scheduleTime),
getMonthAndDateFromEpoch(currentTime),
totalAssetTemplate,
descriptionTemplate,
ownershipTemplate,
@ -190,6 +192,9 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
private DataInsightTotalAssetTemplate createTotalAssetTemplate(
SearchClient searchClient, String team, Long scheduleTime, Long currentTime, int numberOfDays)
throws ParseException, IOException {
// Create A Date Map
Map<String, Integer> dateMap = new LinkedHashMap<>();
Utilities.getLastSevenDays(currentTime).forEach(day -> dateMap.put(day, 0));
// Get total Assets Data
TreeMap<Long, List<Object>> dateWithDataMap =
searchClient.getSortedDate(
@ -199,7 +204,6 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
TOTAL_ENTITIES_BY_TYPE,
ENTITY_REPORT_DATA_INDEX.value());
if (dateWithDataMap.firstEntry() != null && dateWithDataMap.lastEntry() != null) {
List<TotalEntitiesByType> first =
JsonUtils.convertValue(dateWithDataMap.firstEntry().getValue(), new TypeReference<>() {});
List<TotalEntitiesByType> last =
@ -207,16 +211,29 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
Double previousCount = getCountOfEntitiesFromList(first);
Double currentCount = getCountOfEntitiesFromList(last);
dateWithDataMap.forEach(
(key, value) -> {
List<TotalEntitiesByType> list =
JsonUtils.convertValue(value, new TypeReference<>() {});
Double count = getCountOfEntitiesFromList(list);
dateMap.put(Utilities.getDateFromEpoch(key), count.intValue());
});
processDateMapToNormalize(dateMap);
if (previousCount == 0D) {
// it should be undefined
return new DataInsightTotalAssetTemplate(currentCount, 0D, numberOfDays);
return new DataInsightTotalAssetTemplate(currentCount, 0D, numberOfDays, dateMap);
} else {
return new DataInsightTotalAssetTemplate(
currentCount, ((currentCount - previousCount) / previousCount) * 100, numberOfDays);
currentCount,
((currentCount - previousCount) / previousCount) * 100,
numberOfDays,
dateMap);
}
}
throw new IOException(MISSING_DATA);
return new DataInsightTotalAssetTemplate(0D, 0D, numberOfDays, dateMap);
}
private DataInsightDescriptionAndOwnerTemplate createDescriptionTemplate(
@ -226,6 +243,9 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
Long currentTime,
int numberOfDaysChange)
throws ParseException, IOException {
// Create A Date Map
Map<String, Integer> dateMap = new LinkedHashMap<>();
Utilities.getLastSevenDays(currentTime).forEach(day -> dateMap.put(day, 0));
// Get total Assets Data
// This assumes that on a particular date the correct count per entities are given
TreeMap<Long, List<Object>> dateWithDataMap =
@ -246,6 +266,16 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
double currentCompletedDescription = getCompletedDescriptionCount(last);
double currentTotalCount = getTotalEntityFromDescriptionList(last);
dateWithDataMap.forEach(
(key, value) -> {
List<PercentageOfEntitiesWithDescriptionByType> list =
JsonUtils.convertValue(value, new TypeReference<>() {});
Double count = getCompletedDescriptionCount(list);
dateMap.put(Utilities.getDateFromEpoch(key), count.intValue());
});
processDateMapToNormalize(dateMap);
// Previous Percent
double previousPercentCompleted = 0D;
if (previousTotalCount != 0) {
@ -262,10 +292,17 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE,
currentPercentCompleted,
currentPercentCompleted - previousPercentCompleted,
numberOfDaysChange);
numberOfDaysChange,
dateMap);
}
throw new IOException(MISSING_DATA);
return getTemplate(
DataInsightDescriptionAndOwnerTemplate.MetricType.DESCRIPTION,
PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE,
0D,
0D,
numberOfDaysChange,
dateMap);
}
private DataInsightDescriptionAndOwnerTemplate createOwnershipTemplate(
@ -275,6 +312,9 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
Long currentTime,
int numberOfDaysChange)
throws ParseException, IOException {
// Create A Date Map
Map<String, Integer> dateMap = new LinkedHashMap<>();
Utilities.getLastSevenDays(currentTime).forEach(day -> dateMap.put(day, 0));
// Get total Assets Data
// This assumes that on a particular date the correct count per entities are given
TreeMap<Long, List<Object>> dateWithDataMap =
@ -305,16 +345,31 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
if (currentTotalCount != 0) {
currentPercentCompleted = (currentHasOwner / currentTotalCount) * 100;
}
dateWithDataMap.forEach(
(key, value) -> {
List<PercentageOfEntitiesWithOwnerByType> list =
JsonUtils.convertValue(value, new TypeReference<>() {});
Double count = getCompletedOwnershipCount(list);
dateMap.put(Utilities.getDateFromEpoch(key), count.intValue());
});
processDateMapToNormalize(dateMap);
return getTemplate(
DataInsightDescriptionAndOwnerTemplate.MetricType.OWNER,
PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE,
currentPercentCompleted,
currentPercentCompleted - previousPercentCompleted,
numberOfDaysChange);
numberOfDaysChange,
dateMap);
}
throw new IOException(MISSING_DATA);
return getTemplate(
DataInsightDescriptionAndOwnerTemplate.MetricType.OWNER,
PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE,
0D,
0D,
numberOfDaysChange,
dateMap);
}
private DataInsightDescriptionAndOwnerTemplate createTierTemplate(
@ -324,6 +379,9 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
Long currentTime,
int numberOfDaysChange)
throws ParseException, IOException {
// Create A Date Map
Map<String, Integer> dateMap = new LinkedHashMap<>();
Utilities.getLastSevenDays(currentTime).forEach(day -> dateMap.put(day, 0));
// Get total Assets Data
// This assumes that on a particular date the correct count per entities are given
TreeMap<Long, List<Object>> dateWithDataMap =
@ -336,6 +394,14 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
if (dateWithDataMap.lastEntry() != null) {
List<TotalEntitiesByTier> last =
JsonUtils.convertValue(dateWithDataMap.lastEntry().getValue(), new TypeReference<>() {});
dateWithDataMap.forEach(
(key, value) -> {
List<TotalEntitiesByTier> list =
JsonUtils.convertValue(value, new TypeReference<>() {});
Double count = getCountOfTieredEntities(list);
dateMap.put(Utilities.getDateFromEpoch(key), count.intValue());
});
processDateMapToNormalize(dateMap);
Map<String, Double> tierData = getTierData(last);
return new DataInsightDescriptionAndOwnerTemplate(
DataInsightDescriptionAndOwnerTemplate.MetricType.TIER,
@ -346,10 +412,21 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
false,
"",
numberOfDaysChange,
tierData);
tierData,
dateMap);
}
throw new IOException(MISSING_DATA);
return new DataInsightDescriptionAndOwnerTemplate(
DataInsightDescriptionAndOwnerTemplate.MetricType.TIER,
null,
0D,
KPI_NOT_SET,
0D,
false,
"",
numberOfDaysChange,
new HashMap<>(),
dateMap);
}
private Double getCountOfEntitiesFromList(List<TotalEntitiesByType> entitiesByTypeList) {
@ -361,6 +438,15 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
return totalCount;
}
private Double getCountOfTieredEntities(List<TotalEntitiesByTier> entitiesByTierList) {
// If there are multiple entries for same entities then this can yield invalid results
double totalCount = 0D;
for (TotalEntitiesByTier obj : entitiesByTierList) {
totalCount += obj.getEntityCountFraction() * 100;
}
return totalCount;
}
private Map<String, Double> getTierData(List<TotalEntitiesByTier> entitiesByTypeList) {
// If there are multiple entries for same entities then this can yield invalid results
Map<String, Double> data = new TreeMap<>();
@ -415,7 +501,8 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
DataInsightChartResult.DataInsightChartType chartType,
Double percentCompleted,
Double percentChange,
int numberOfDaysChange) {
int numberOfDaysChange,
Map<String, Integer> dateMap) {
List<Kpi> kpiList = getAvailableKpi();
Kpi validKpi = null;
@ -463,60 +550,32 @@ public class DataInsightsReportApp extends AbstractNativeApplication {
isKpiAvailable,
totalDaysLeft,
numberOfDaysChange,
null);
null,
dateMap);
}
private long getTimeFromSchedule(
AppSchedule appSchedule, JobExecutionContext jobExecutionContext) {
AppSchedule.ScheduleTimeline timeline = appSchedule.getScheduleType();
return switch (timeline) {
case HOURLY -> 3600000L;
case DAILY -> 86400000L;
case WEEKLY -> 604800000L;
case MONTHLY -> 2592000000L;
case CUSTOM -> {
if (jobExecutionContext.getTrigger() != null) {
Trigger triggerQrz = jobExecutionContext.getTrigger();
Date previousFire =
triggerQrz.getPreviousFireTime() == null
? triggerQrz.getStartTime()
: triggerQrz.getPreviousFireTime();
yield previousFire.toInstant().toEpochMilli();
}
yield 86400000L;
}
};
private void processDateMapToNormalize(Map<String, Integer> dateMap) {
Pair<Integer, Integer> maxIn = getMinAndMax(dateMap.values().stream().toList());
dateMap.replaceAll(
(k, v) ->
getNormalizedValue(
v.doubleValue(), maxIn.getRight().doubleValue(), maxIn.getLeft().doubleValue())
.intValue());
}
public static int getNumberOfDays(AppSchedule appSchedule) {
AppSchedule.ScheduleTimeline timeline = appSchedule.getScheduleType();
switch (timeline) {
case HOURLY:
return 0;
case DAILY:
return 1;
case WEEKLY:
return 7;
case MONTHLY:
return 30;
case CUSTOM:
if (!CommonUtil.nullOrEmpty(appSchedule.getCronExpression())) {
Trigger triggerQrz =
CronScheduleBuilder.cronSchedule(appSchedule.getCronExpression()).build();
Date previousFire =
triggerQrz.getPreviousFireTime() == null
? triggerQrz.getStartTime()
: triggerQrz.getPreviousFireTime();
Date nextFire = triggerQrz.getFireTimeAfter(previousFire);
Period period =
Period.between(
previousFire.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(),
nextFire.toInstant().atZone(ZoneId.systemDefault()).toLocalDate());
return period.getDays();
} else {
throw new IllegalArgumentException("Missing Cron Expression for Custom Schedule.");
}
private Pair<Integer, Integer> getMinAndMax(List<Integer> integers) {
Optional<Integer> minOptional = integers.stream().min(Integer::compareTo);
Optional<Integer> maxOptional = integers.stream().max(Integer::compareTo);
int min = minOptional.orElseThrow(() -> new IllegalArgumentException("List is empty"));
int max = maxOptional.orElseThrow(() -> new IllegalArgumentException("List is empty"));
return Pair.of(min, max);
}
private Double getNormalizedValue(Double value, Double max, Double min) {
if (max - min == 0) {
return 0d;
}
throw new IllegalArgumentException("Invalid Trigger Type, Can only be Scheduled.");
return ((value - min) / (max - min) * 50);
}
}

1
openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java
View File

@ -48,6 +48,7 @@ import org.openmetadata.service.workflows.searchIndex.PaginatedEntitiesSource;
import org.quartz.JobExecutionContext;
@Slf4j
@SuppressWarnings("unused")
public class SearchIndexApp extends AbstractNativeApplication {
private static final String ALL = "all";

1
openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/test/NoOpTestApplication.java
View File

@ -7,6 +7,7 @@ import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.search.SearchRepository;
@Slf4j
@SuppressWarnings("unused")
public class NoOpTestApplication extends AbstractNativeApplication {
@Override

16
openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightDescriptionAndOwnerTemplate.java
View File

@ -15,6 +15,7 @@ package org.openmetadata.service.events.scheduled.template;
import java.util.Map;
@SuppressWarnings("unused")
public class DataInsightDescriptionAndOwnerTemplate {
public enum MetricType {
DESCRIPTION,
@ -36,6 +37,7 @@ public class DataInsightDescriptionAndOwnerTemplate {
private String completeMessage;
private int numberOfDaysChange;
private Map<String, Double> tierMap;
private Map<String, Integer> dateMap;
public DataInsightDescriptionAndOwnerTemplate(
MetricType metricType,
@ -46,7 +48,8 @@ public class DataInsightDescriptionAndOwnerTemplate {
boolean isKpiAvailable,
String numberOfDaysLeft,
int numberOfDaysChange,
Map<String, Double> tierMap) {
Map<String, Double> tierMap,
Map<String, Integer> dateMap) {
this.percentCompleted = String.format("%.2f", percentCompleted);
this.targetKpi = targetKpi;
this.percentChange = String.format("%.2f", percentChange);
@ -54,6 +57,7 @@ public class DataInsightDescriptionAndOwnerTemplate {
this.numberOfDaysLeft = numberOfDaysLeft;
this.tierMap = tierMap;
this.numberOfDaysChange = numberOfDaysChange;
this.dateMap = dateMap;
String color = "#BF0000";
if (percentChange > 0) {
color = "#008510";
@ -86,7 +90,7 @@ public class DataInsightDescriptionAndOwnerTemplate {
case NOT_MET -> "The Target set for KPIs was not met it’s time to restructure your goals and progress faster.";
};
}
return "You have not set any KPIS yet, it’s time to restructure your goals, set KPIs and progress faster.";
return "You have not set any KPIs yet, it’s time to restructure your goals, set KPIs and progress faster.";
}
return "";
}
@ -150,4 +154,12 @@ public class DataInsightDescriptionAndOwnerTemplate {
public void setNumberOfDaysChange(int numberOfDaysChange) {
this.numberOfDaysChange = numberOfDaysChange;
}
public Map<String, Integer> getDateMap() {
return dateMap;
}
public void setDateMap(Map<String, Integer> dateMap) {
this.dateMap = dateMap;
}
}

18
openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightTotalAssetTemplate.java
View File

@ -13,17 +13,25 @@
package org.openmetadata.service.events.scheduled.template;
import java.util.Map;
@SuppressWarnings("unused")
public class DataInsightTotalAssetTemplate {
private String totalDataAssets;
private String percentChangeTotalAssets;
private String completeMessage;
private int numberOfDaysChange;
private Map<String, Integer> dateMap;
public DataInsightTotalAssetTemplate(
Double totalDataAssets, Double percentChangeTotalAssets, int numberOfDaysChange) {
Double totalDataAssets,
Double percentChangeTotalAssets,
int numberOfDaysChange,
Map<String, Integer> dateMap) {
this.totalDataAssets = String.format("%.2f", totalDataAssets);
this.percentChangeTotalAssets = String.format("%.2f", percentChangeTotalAssets);
this.numberOfDaysChange = numberOfDaysChange;
this.dateMap = dateMap;
String color = "#BF0000";
if (percentChangeTotalAssets > 0) {
color = "#008510";
@ -65,4 +73,12 @@ public class DataInsightTotalAssetTemplate {
public void setNumberOfDaysChange(int numberOfDaysChange) {
this.numberOfDaysChange = numberOfDaysChange;
}
public Map<String, Integer> getDateMap() {
return dateMap;
}
public void setDateMap(Map<String, Integer> dateMap) {
this.dateMap = dateMap;
}
}

4
openmetadata-service/src/main/java/org/openmetadata/service/util/EmailUtil.java
View File

@ -337,6 +337,8 @@ public class EmailUtil {
public static void sendDataInsightEmailNotificationToUser(
Set<String> emails,
String startDate,
String endDate,
DataInsightTotalAssetTemplate totalAssetObj,
DataInsightDescriptionAndOwnerTemplate descriptionObj,
DataInsightDescriptionAndOwnerTemplate ownerShipObj,
@ -346,6 +348,8 @@ public class EmailUtil {
throws IOException, TemplateException {
if (Boolean.TRUE.equals(getSmtpSettings().getEnableSmtpServer())) {
Map<String, Object> templatePopulator = new HashMap<>();
templatePopulator.put("startDate", startDate);
templatePopulator.put("endDate", endDate);
templatePopulator.put("totalAssetObj", totalAssetObj);
templatePopulator.put("descriptionObj", descriptionObj);
templatePopulator.put("ownershipObj", ownerShipObj);

50
openmetadata-service/src/main/java/org/openmetadata/service/util/Utilities.java Normal file
View File

@ -0,0 +1,50 @@
package org.openmetadata.service.util;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
public class Utilities {
private Utilities() {}
public static List<String> getLastSevenDays(long currentEpochTimestampInMilli) {
List<String> lastSevenDays = new ArrayList<>();
// Create a formatter for the date
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("d");
// Calculate and add the dates for the last seven days
for (int i = 6; i >= 0; i--) {
long dayEpochTimestamp =
currentEpochTimestampInMilli
- ((long) i * 24 * 60 * 60 * 1000); // Subtracting seconds for each day
LocalDateTime dateTime =
LocalDateTime.ofInstant(
Instant.ofEpochMilli(dayEpochTimestamp), java.time.ZoneId.systemDefault());
lastSevenDays.add(dateTime.format(formatter));
}
return lastSevenDays;
}
public static String getMonthAndDateFromEpoch(long epochTimestamp) {
return getFormattedDateFromEpoch(epochTimestamp, "MMM d");
}
public static String getDateFromEpoch(long epochTimestampInMilli) {
return getFormattedDateFromEpoch(epochTimestampInMilli, "d");
}
private static String getFormattedDateFromEpoch(long epochTimestamp, String format) {
Instant instant = Instant.ofEpochMilli(epochTimestamp);
LocalDateTime dateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault());
// Define a custom date formatter
DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern(format);
return dateTime.format(dateFormat);
}
}

1181
openmetadata-service/src/main/resources/emailTemplates/dataInsightReport.ftl
View File

File diff suppressed because it is too large Load Diff