diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchEventHandler.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchEventHandler.java index 86aa587e59f..e0aa49693a9 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchEventHandler.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchEventHandler.java @@ -133,9 +133,11 @@ public class ElasticSearchEventHandler implements EventHandler { ElasticSearchIndexType esIndexType = esIndexDefinition.getIndexMappingByEntityType(entityType); UUID entityId = event.getEntityId(); ChangeDescription changeDescription = event.getChangeDescription(); + List fieldsAdded = changeDescription.getFieldsAdded(); StringBuilder scriptTxt = new StringBuilder(); Map fieldAddParams = new HashMap<>(); + for (FieldChange fieldChange: fieldsAdded) { if (fieldChange.getName().equalsIgnoreCase("followers")) { List entityReferences = (List) fieldChange.getNewValue(); @@ -154,17 +156,25 @@ public class ElasticSearchEventHandler implements EventHandler { for (EntityReference follower : entityReferences) { fieldAddParams.put(fieldChange.getName(), follower.getId().toString()); } - - scriptTxt.append("ctx._source.followers.removeAll(Collections.singleton(params.followers))"); + scriptTxt.append("ctx._source.followers.removeAll(Collections.singleton(params.followers));"); } } - + ESChangeDescription esChangeDescription = ESChangeDescription.builder() + .updatedAt(event.getDateTime().getTime()) + .updatedBy(event.getUserName()).build(); + esChangeDescription.setFieldsAdded(changeDescription.getFieldsAdded()); + esChangeDescription.setFieldsDeleted(changeDescription.getFieldsDeleted()); + esChangeDescription.setFieldsUpdated(changeDescription.getFieldsUpdated()); + Map esChangeDescriptionDoc = JsonUtils.getMap(esChangeDescription); + fieldAddParams.put("change_description", esChangeDescriptionDoc); + scriptTxt.append("ctx._source.change_descriptions.add(params.change_description);"); if (!scriptTxt.toString().isEmpty()) { Script script = new Script(ScriptType.INLINE, "painless", scriptTxt.toString(), fieldAddParams); UpdateRequest updateRequest = new UpdateRequest(esIndexType.indexName, entityId.toString()); updateRequest.script(script); + return updateRequest; } else { return null; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java index ec132e5991a..b286f868c6a 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java @@ -189,27 +189,6 @@ class ElasticSearchIndex { Long lastUpdatedTimestamp = System.currentTimeMillis(); @JsonProperty("change_descriptions") List changeDescriptions; - - - public void parseTags(List tags) { - if (!tags.isEmpty()) { - List tagsList = new ArrayList<>(tags); - String tierTag = null; - for (String tag : tagsList) { - if (tag.toLowerCase().matches("(.*)tier(.*)")) { - tierTag = tag; - break; - } - } - if (tierTag != null) { - tagsList.remove(tierTag); - this.tier = tierTag; - } - this.tags = tagsList; - } else { - this.tags = tags; - } - } } @Getter @@ -238,6 +217,31 @@ class ESChangeDescription { List fieldsDeleted; } +class ParseTags { + String tierTag; + List tags; + + ParseTags(List tags) { + if (!tags.isEmpty()) { + List tagsList = new ArrayList<>(tags); + String tierTag = null; + for (String tag : tagsList) { + if (tag.toLowerCase().matches("(.*)tier(.*)")) { + tierTag = tag; + break; + } + } + if (tierTag != null) { + tagsList.remove(tierTag); + this.tierTag = tierTag; + } + this.tags = tagsList; + } else { + this.tags = tags; + } + } +} + @EqualsAndHashCode(callSuper = true) @Getter @SuperBuilder(builderMethodName = "internalBuilder") @@ -294,6 +298,7 @@ class TableESIndex extends ElasticSearchIndex { columnNames.add(col.getName()); } } + ParseTags parseTags = new ParseTags(tags); TableESIndexBuilder tableESIndexBuilder = internalBuilder().tableId(tableId) .name(tableName) .displayName(tableName) @@ -305,7 +310,8 @@ class TableESIndex extends ElasticSearchIndex { .columnNames(columnNames) .columnDescriptions(columnDescriptions) .tableType(table.getTableType().toString()) - .tags(tags); + .tags(parseTags.tags) + .tier(parseTags.tierTag); if (table.getDatabase() != null) { tableESIndexBuilder.database(table.getDatabase().getName()); @@ -400,7 +406,7 @@ class TopicESIndex extends ElasticSearchIndex { if (topic.getTags() != null) { topic.getTags().forEach(tag -> tags.add(tag.getTagFQN())); } - + ParseTags parseTags = new ParseTags(tags); TopicESIndexBuilder topicESIndexBuilder = internalBuilder().topicId(topic.getId().toString()) .name(topic.getName()) .displayName(topic.getDisplayName()) @@ -411,7 +417,8 @@ class TopicESIndex extends ElasticSearchIndex { .serviceType(topic.getServiceType().toString()) .serviceCategory("messagingService") .entityType("topic") - .tags(tags); + .tags(parseTags.tags) + .tier(parseTags.tierTag); if (topic.getFollowers() != null) { topicESIndexBuilder.followers(topic.getFollowers().stream().map(item -> @@ -485,7 +492,7 @@ class DashboardESIndex extends ElasticSearchIndex { chartNames.add(chart.getDisplayName()); chartDescriptions.add(chart.getDescription()); } - + ParseTags parseTags = new ParseTags(tags); DashboardESIndexBuilder dashboardESIndexBuilder = internalBuilder().dashboardId(dashboard.getId().toString()) .name(dashboard.getDisplayName()) .displayName(dashboard.getDisplayName()) @@ -498,7 +505,8 @@ class DashboardESIndex extends ElasticSearchIndex { .service(dashboard.getService().getName()) .serviceType(dashboard.getServiceType().toString()) .serviceCategory("dashboardService") - .tags(tags); + .tags(parseTags.tags) + .tier(parseTags.tierTag); if (dashboard.getUsageSummary() != null) { dashboardESIndexBuilder.weeklyStats(dashboard.getUsageSummary().getWeeklyStats().getCount()) @@ -566,7 +574,7 @@ class PipelineESIndex extends ElasticSearchIndex { taskNames.add(task.getDisplayName()); taskDescriptions.add(task.getDescription()); } - + ParseTags parseTags = new ParseTags(tags); PipelineESIndexBuilder pipelineESIndexBuilder = internalBuilder().pipelineId(pipeline.getId().toString()) .name(pipeline.getDisplayName()) .displayName(pipeline.getDisplayName()) @@ -579,7 +587,8 @@ class PipelineESIndex extends ElasticSearchIndex { .service(pipeline.getService().getName()) .serviceType(pipeline.getServiceType().toString()) .serviceCategory("pipelineService") - .tags(tags); + .tags(parseTags.tags) + .tier(parseTags.tierTag); if (pipeline.getFollowers() != null) { pipelineESIndexBuilder.followers(pipeline.getFollowers().stream().map(item -> @@ -608,4 +617,4 @@ class PipelineESIndex extends ElasticSearchIndex { pipelineESIndexBuilder.changeDescriptions(esChangeDescription != null ? List.of(esChangeDescription) : null); return pipelineESIndexBuilder; } -} \ No newline at end of file +}