diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java index dd0997a7748..8782ce2c51d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java @@ -21,9 +21,14 @@ import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.system.IndexingError; import org.openmetadata.schema.system.StepStats; +import org.openmetadata.schema.type.ChangeDescription; +import org.openmetadata.schema.type.ChangeSummaryMap; +import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.schema.type.change.ChangeSource; +import org.openmetadata.schema.type.change.ChangeSummary; import org.openmetadata.service.Entity; import org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils; import org.openmetadata.service.exception.EntityNotFoundException; @@ -145,7 +150,12 @@ public class DataInsightsEntityEnricherProcessor entityMap.keySet().retainAll((List) contextData.get(ENTITY_TYPE_FIELDS_KEY)); String entityType = (String) contextData.get(ENTITY_TYPE_KEY); - List> interfaces = List.of(entity.getClass().getInterfaces()); + + Map changeSummaryMap = + Optional.ofNullable(entity.getChangeDescription()) + .map(ChangeDescription::getChangeSummary) + .map(ChangeSummaryMap::getAdditionalProperties) + .orElse(null); // Enrich with EntityType if (CommonUtil.nullOrEmpty(entityType)) { @@ -159,13 +169,105 @@ public class DataInsightsEntityEnricherProcessor entityMap.put("startTimestamp", startTimestamp); entityMap.put("endTimestamp", endTimestamp); - // Enrich with Team + // Process Description Source + entityMap.put("descriptionSources", processDescriptionSources(entity, changeSummaryMap)); + + // Process Tag Source + entityMap.put("tagSources", processTagSources(entity)); + + // Process Team + Optional.ofNullable(processTeam(entity)).ifPresent(team -> entityMap.put("team", team)); + + // Process Tier + Optional.ofNullable(processTier(entity)).ifPresent(tier -> entityMap.put("tier", tier)); + + // Enrich with Description Stats + entityMap.put("hasDescription", CommonUtil.nullOrEmpty(entity.getDescription()) ? 0 : 1); + + if (hasColumns(entity)) { + entityMap.put("numberOfColumns", ((ColumnsEntityInterface) entity).getColumns().size()); + entityMap.put( + "numberOfColumnsWithDescription", + ((ColumnsEntityInterface) entity) + .getColumns().stream() + .map(column -> CommonUtil.nullOrEmpty(column.getDescription()) ? 0 : 1) + .reduce(0, Integer::sum)); + } + + // Modify Custom Property key + Optional oCustomProperties = Optional.ofNullable(entityMap.get("extension")); + oCustomProperties.ifPresent( + o -> entityMap.put(String.format("%sCustomProperty", entityType), o)); + + return entityMap; + } + + private boolean hasColumns(EntityInterface entity) { + return List.of(entity.getClass().getInterfaces()).contains(ColumnsEntityInterface.class); + } + + private String getDescriptionSource( + String description, Map changeSummaryMap, String changeSummaryKey) { + if (description == null) { + return null; + } + + String descriptionSource = ChangeSource.INGESTED.value(); + + if (changeSummaryMap != null) { + if (changeSummaryMap.containsKey(changeSummaryKey)) { + descriptionSource = changeSummaryMap.get(changeSummaryKey).getChangeSource().value(); + } + } + return descriptionSource; + } + + private void processDescriptionSource( + EntityInterface entity, + Map changeSummaryMap, + Map descriptionSources) { + Optional.ofNullable( + getDescriptionSource(entity.getDescription(), changeSummaryMap, "description")) + .ifPresent( + source -> + descriptionSources.put(source, descriptionSources.getOrDefault(source, 0) + 1)); + } + + private void processColumnDescriptionSources( + ColumnsEntityInterface entity, + Map changeSummaryMap, + Map descriptionSources) { + for (Column column : entity.getColumns()) { + Optional.ofNullable( + getDescriptionSource( + column.getDescription(), + changeSummaryMap, + String.format("columns.%s.description", column.getName()))) + .ifPresent( + source -> + descriptionSources.put(source, descriptionSources.getOrDefault(source, 0) + 1)); + } + } + + private Map processDescriptionSources( + EntityInterface entity, Map changeSummaryMap) { + Map descriptionSources = new HashMap<>(); + processDescriptionSource(entity, changeSummaryMap, descriptionSources); + if (hasColumns(entity)) { + processColumnDescriptionSources( + (ColumnsEntityInterface) entity, changeSummaryMap, descriptionSources); + } + return descriptionSources; + } + + private String processTeam(EntityInterface entity) { + String team = null; Optional> oEntityOwners = Optional.ofNullable(entity.getOwners()); if (oEntityOwners.isPresent() && !oEntityOwners.get().isEmpty()) { EntityReference entityOwner = oEntityOwners.get().get(0); String ownerType = entityOwner.getType(); if (ownerType.equals(Entity.TEAM)) { - entityMap.put("team", entityOwner.getName()); + team = entityOwner.getName(); } else { try { Optional oOwner = @@ -178,7 +280,7 @@ public class DataInsightsEntityEnricherProcessor List teams = owner.getTeams(); if (!teams.isEmpty()) { - entityMap.put("team", teams.get(0).getName()); + team = teams.get(0).getName(); } } } catch (EntityNotFoundException ex) { @@ -188,50 +290,65 @@ public class DataInsightsEntityEnricherProcessor String.format( "Owner %s for %s '%s' version '%s' not found.", entityOwner.getFullyQualifiedName(), - entityType, + Entity.getEntityTypeFromObject(entity), entity.getFullyQualifiedName(), entity.getVersion())); } } } + return team; + } + + private void processTagSources(List tagList, Map tagSources) { + Optional.ofNullable(tagList) + .ifPresent( + tags -> { + tags.stream() + .filter(tag -> !tag.getTagFQN().startsWith("Tier.")) + .map(tag -> tag.getLabelType().value()) + .forEach( + tagSource -> + tagSources.put(tagSource, tagSources.getOrDefault(tagSource, 0) + 1)); + }); + } + + private void processEntityTagSources(EntityInterface entity, Map tagSources) { + processTagSources(entity.getTags(), tagSources); + } + + private void processColumnTagSources( + ColumnsEntityInterface entity, Map tagSources) { + for (Column column : entity.getColumns()) { + processTagSources(column.getTags(), tagSources); + } + } + + private Map processTagSources(EntityInterface entity) { + Map tagSources = new HashMap<>(); + processEntityTagSources(entity, tagSources); + if (hasColumns(entity)) { + processColumnTagSources((ColumnsEntityInterface) entity, tagSources); + } + return tagSources; + } + + private String processTier(EntityInterface entity) { + String tier = null; + + if (!NON_TIER_ENTITIES.contains(Entity.getEntityTypeFromObject(entity))) { + tier = "NoTier"; + } - // Enrich with Tier Optional> oEntityTags = Optional.ofNullable(entity.getTags()); if (oEntityTags.isPresent()) { Optional oEntityTier = getEntityTier(oEntityTags.get().stream().map(TagLabel::getTagFQN).toList()); - oEntityTier.ifPresentOrElse( - s -> entityMap.put("tier", s), - () -> { - if (!NON_TIER_ENTITIES.contains(entityType)) { - entityMap.put("tier", "NoTier"); - } - }); - } else if (!NON_TIER_ENTITIES.contains(entityType)) { - entityMap.put("tier", "NoTier"); + if (oEntityTier.isPresent()) { + tier = oEntityTier.get(); + } } - - // Enrich with Description Stats - if (interfaces.contains(ColumnsEntityInterface.class)) { - entityMap.put("numberOfColumns", ((ColumnsEntityInterface) entity).getColumns().size()); - entityMap.put( - "numberOfColumnsWithDescription", - ((ColumnsEntityInterface) entity) - .getColumns().stream() - .map(column -> CommonUtil.nullOrEmpty(column.getDescription()) ? 0 : 1) - .reduce(0, Integer::sum)); - entityMap.put("hasDescription", CommonUtil.nullOrEmpty(entity.getDescription()) ? 0 : 1); - } else { - entityMap.put("hasDescription", CommonUtil.nullOrEmpty(entity.getDescription()) ? 0 : 1); - } - - // Modify Custom Property key - Optional oCustomProperties = Optional.ofNullable(entityMap.get("extension")); - oCustomProperties.ifPresent( - o -> entityMap.put(String.format("%sCustomProperty", entityType), o)); - - return entityMap; + return tier; } private Optional getEntityTier(List entityTags) {