Add tag and description Sources to Data Insights (#20087)

openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java

@@ -21,9 +21,14 @@ import org.openmetadata.schema.EntityInterface;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.system.IndexingError;
 import org.openmetadata.schema.system.StepStats;
+import org.openmetadata.schema.type.ChangeDescription;
+import org.openmetadata.schema.type.ChangeSummaryMap;
+import org.openmetadata.schema.type.Column;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.schema.type.TagLabel;
+import org.openmetadata.schema.type.change.ChangeSource;
+import org.openmetadata.schema.type.change.ChangeSummary;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils;
 import org.openmetadata.service.exception.EntityNotFoundException;
@@ -145,7 +150,12 @@ public class DataInsightsEntityEnricherProcessor
     entityMap.keySet().retainAll((List<String>) contextData.get(ENTITY_TYPE_FIELDS_KEY));
 
     String entityType = (String) contextData.get(ENTITY_TYPE_KEY);
-    List<Class<?>> interfaces = List.of(entity.getClass().getInterfaces());
+
+    Map<String, ChangeSummary> changeSummaryMap =
+        Optional.ofNullable(entity.getChangeDescription())
+            .map(ChangeDescription::getChangeSummary)
+            .map(ChangeSummaryMap::getAdditionalProperties)
+            .orElse(null);
 
     // Enrich with EntityType
     if (CommonUtil.nullOrEmpty(entityType)) {
@@ -159,13 +169,105 @@ public class DataInsightsEntityEnricherProcessor
     entityMap.put("startTimestamp", startTimestamp);
     entityMap.put("endTimestamp", endTimestamp);
 
-    // Enrich with Team
+    // Process Description Source
+    entityMap.put("descriptionSources", processDescriptionSources(entity, changeSummaryMap));
+
+    // Process Tag Source
+    entityMap.put("tagSources", processTagSources(entity));
+
+    // Process Team
+    Optional.ofNullable(processTeam(entity)).ifPresent(team -> entityMap.put("team", team));
+
+    // Process Tier
+    Optional.ofNullable(processTier(entity)).ifPresent(tier -> entityMap.put("tier", tier));
+
+    // Enrich with Description Stats
+    entityMap.put("hasDescription", CommonUtil.nullOrEmpty(entity.getDescription()) ? 0 : 1);
+
+    if (hasColumns(entity)) {
+      entityMap.put("numberOfColumns", ((ColumnsEntityInterface) entity).getColumns().size());
+      entityMap.put(
+          "numberOfColumnsWithDescription",
+          ((ColumnsEntityInterface) entity)
+              .getColumns().stream()
+                  .map(column -> CommonUtil.nullOrEmpty(column.getDescription()) ? 0 : 1)
+                  .reduce(0, Integer::sum));
+    }
+
+    // Modify Custom Property key
+    Optional<Object> oCustomProperties = Optional.ofNullable(entityMap.get("extension"));
+    oCustomProperties.ifPresent(
+        o -> entityMap.put(String.format("%sCustomProperty", entityType), o));
+
+    return entityMap;
+  }
+
+  private boolean hasColumns(EntityInterface entity) {
+    return List.of(entity.getClass().getInterfaces()).contains(ColumnsEntityInterface.class);
+  }
+
+  private String getDescriptionSource(
+      String description, Map<String, ChangeSummary> changeSummaryMap, String changeSummaryKey) {
+    if (description == null) {
+      return null;
+    }
+
+    String descriptionSource = ChangeSource.INGESTED.value();
+
+    if (changeSummaryMap != null) {
+      if (changeSummaryMap.containsKey(changeSummaryKey)) {
+        descriptionSource = changeSummaryMap.get(changeSummaryKey).getChangeSource().value();
+      }
+    }
+    return descriptionSource;
+  }
+
+  private void processDescriptionSource(
+      EntityInterface entity,
+      Map<String, ChangeSummary> changeSummaryMap,
+      Map<String, Integer> descriptionSources) {
+    Optional.ofNullable(
+            getDescriptionSource(entity.getDescription(), changeSummaryMap, "description"))
+        .ifPresent(
+            source ->
+                descriptionSources.put(source, descriptionSources.getOrDefault(source, 0) + 1));
+  }
+
+  private void processColumnDescriptionSources(
+      ColumnsEntityInterface entity,
+      Map<String, ChangeSummary> changeSummaryMap,
+      Map<String, Integer> descriptionSources) {
+    for (Column column : entity.getColumns()) {
+      Optional.ofNullable(
+              getDescriptionSource(
+                  column.getDescription(),
+                  changeSummaryMap,
+                  String.format("columns.%s.description", column.getName())))
+          .ifPresent(
+              source ->
+                  descriptionSources.put(source, descriptionSources.getOrDefault(source, 0) + 1));
+    }
+  }
+
+  private Map<String, Integer> processDescriptionSources(
+      EntityInterface entity, Map<String, ChangeSummary> changeSummaryMap) {
+    Map<String, Integer> descriptionSources = new HashMap<>();
+    processDescriptionSource(entity, changeSummaryMap, descriptionSources);
+    if (hasColumns(entity)) {
+      processColumnDescriptionSources(
+          (ColumnsEntityInterface) entity, changeSummaryMap, descriptionSources);
+    }
+    return descriptionSources;
+  }
+
+  private String processTeam(EntityInterface entity) {
+    String team = null;
     Optional<List<EntityReference>> oEntityOwners = Optional.ofNullable(entity.getOwners());
     if (oEntityOwners.isPresent() && !oEntityOwners.get().isEmpty()) {
       EntityReference entityOwner = oEntityOwners.get().get(0);
       String ownerType = entityOwner.getType();
       if (ownerType.equals(Entity.TEAM)) {
-        entityMap.put("team", entityOwner.getName());
+        team = entityOwner.getName();
       } else {
         try {
           Optional<User> oOwner =
@@ -178,7 +280,7 @@ public class DataInsightsEntityEnricherProcessor
             List<EntityReference> teams = owner.getTeams();
 
             if (!teams.isEmpty()) {
-              entityMap.put("team", teams.get(0).getName());
+              team = teams.get(0).getName();
             }
           }
         } catch (EntityNotFoundException ex) {
@@ -188,50 +290,65 @@ public class DataInsightsEntityEnricherProcessor
               String.format(
                   "Owner %s for %s '%s' version '%s' not found.",
                   entityOwner.getFullyQualifiedName(),
-                  entityType,
+                  Entity.getEntityTypeFromObject(entity),
                   entity.getFullyQualifiedName(),
                   entity.getVersion()));
         }
       }
     }
+    return team;
+  }
+
+  private void processTagSources(List<TagLabel> tagList, Map<String, Integer> tagSources) {
+    Optional.ofNullable(tagList)
+        .ifPresent(
+            tags -> {
+              tags.stream()
+                  .filter(tag -> !tag.getTagFQN().startsWith("Tier."))
+                  .map(tag -> tag.getLabelType().value())
+                  .forEach(
+                      tagSource ->
+                          tagSources.put(tagSource, tagSources.getOrDefault(tagSource, 0) + 1));
+            });
+  }
+
+  private void processEntityTagSources(EntityInterface entity, Map<String, Integer> tagSources) {
+    processTagSources(entity.getTags(), tagSources);
+  }
+
+  private void processColumnTagSources(
+      ColumnsEntityInterface entity, Map<String, Integer> tagSources) {
+    for (Column column : entity.getColumns()) {
+      processTagSources(column.getTags(), tagSources);
+    }
+  }
+
+  private Map<String, Integer> processTagSources(EntityInterface entity) {
+    Map<String, Integer> tagSources = new HashMap<>();
+    processEntityTagSources(entity, tagSources);
+    if (hasColumns(entity)) {
+      processColumnTagSources((ColumnsEntityInterface) entity, tagSources);
+    }
+    return tagSources;
+  }
+
+  private String processTier(EntityInterface entity) {
+    String tier = null;
+
+    if (!NON_TIER_ENTITIES.contains(Entity.getEntityTypeFromObject(entity))) {
+      tier = "NoTier";
+    }
 
-    // Enrich with Tier
     Optional<List<TagLabel>> oEntityTags = Optional.ofNullable(entity.getTags());
 
     if (oEntityTags.isPresent()) {
       Optional<String> oEntityTier =
           getEntityTier(oEntityTags.get().stream().map(TagLabel::getTagFQN).toList());
-      oEntityTier.ifPresentOrElse(
+      if (oEntityTier.isPresent()) {
-          s -> entityMap.put("tier", s),
+        tier = oEntityTier.get();
-          () -> {
+      }
-            if (!NON_TIER_ENTITIES.contains(entityType)) {
-              entityMap.put("tier", "NoTier");
-            }
-          });
-    } else if (!NON_TIER_ENTITIES.contains(entityType)) {
-      entityMap.put("tier", "NoTier");
     }
-
+    return tier;
-    // Enrich with Description Stats
-    if (interfaces.contains(ColumnsEntityInterface.class)) {
-      entityMap.put("numberOfColumns", ((ColumnsEntityInterface) entity).getColumns().size());
-      entityMap.put(
-          "numberOfColumnsWithDescription",
-          ((ColumnsEntityInterface) entity)
-              .getColumns().stream()
-                  .map(column -> CommonUtil.nullOrEmpty(column.getDescription()) ? 0 : 1)
-                  .reduce(0, Integer::sum));
-      entityMap.put("hasDescription", CommonUtil.nullOrEmpty(entity.getDescription()) ? 0 : 1);
-    } else {
-      entityMap.put("hasDescription", CommonUtil.nullOrEmpty(entity.getDescription()) ? 0 : 1);
-    }
-
-    // Modify Custom Property key
-    Optional<Object> oCustomProperties = Optional.ofNullable(entityMap.get("extension"));
-    oCustomProperties.ifPresent(
-        o -> entityMap.put(String.format("%sCustomProperty", entityType), o));
-
-    return entityMap;
   }
 
   private Optional<String> getEntityTier(List<String> entityTags) {