diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 2db930e8889..868948347cf 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -227,6 +227,11 @@ public class ElasticSearchClient implements SearchClient { "fqnParts", "chart_suggest", "field_suggest"); + private static final List SOURCE_FIELDS_TO_EXCLUDE = + Stream.concat( + FIELDS_TO_REMOVE.stream(), + Stream.of("schemaDefinition", "testSuite", "customMetrics")) + .toList(); static { SearchModule searchModule = new SearchModule(Settings.EMPTY, false, List.of()); @@ -705,13 +710,15 @@ public class ElasticSearchClient implements SearchClient { new es.org.elasticsearch.action.search.SearchRequest( Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + List sourceFieldsToExcludeCopy = new ArrayList<>(SOURCE_FIELDS_TO_EXCLUDE); + sourceFieldsToExcludeCopy.add("lineage"); + searchSourceBuilder.fetchSource(null, sourceFieldsToExcludeCopy.toArray(String[]::new)); searchSourceBuilder.query( QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn))); searchRequest.source(searchSourceBuilder.size(1000)); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); for (var hit : searchResponse.getHits().getHits()) { Map tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); - tempMap.keySet().removeAll(FIELDS_TO_REMOVE); responseMap.put("entity", tempMap); } getLineage( @@ -772,6 +779,7 @@ public class ElasticSearchClient implements SearchClient { new es.org.elasticsearch.action.search.SearchRequest( Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new)); searchSourceBuilder.query( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn)))); @@ -788,7 +796,7 @@ public class ElasticSearchClient implements SearchClient { List> lineage = (List>) hit.getSourceAsMap().get("lineage"); HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); - tempMap.keySet().removeAll(FIELDS_TO_REMOVE); + tempMap.remove("lineage"); nodes.add(tempMap); for (Map lin : lineage) { Map fromEntity = (HashMap) lin.get("fromEntity"); @@ -963,6 +971,7 @@ public class ElasticSearchClient implements SearchClient { QueryBuilders.boolQuery() .must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn))); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new)); searchSourceBuilder.query(boolQueryBuilder); if (CommonUtil.nullOrEmpty(deleted)) { searchSourceBuilder.query( @@ -977,7 +986,7 @@ public class ElasticSearchClient implements SearchClient { List> lineage = (List>) hit.getSourceAsMap().get("lineage"); HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); - tempMap.keySet().removeAll(FIELDS_TO_REMOVE); + tempMap.remove("lineage"); nodes.add(tempMap); for (Map lin : lineage) { HashMap fromEntity = (HashMap) lin.get("fromEntity"); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 0d383edc09a..f03e3138f12 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -221,6 +221,11 @@ public class OpenSearchClient implements SearchClient { "fqnParts", "chart_suggest", "field_suggest"); + private static final List SOURCE_FIELDS_TO_EXCLUDE = + Stream.concat( + FIELDS_TO_REMOVE.stream(), + Stream.of("schemaDefinition", "testSuite", "customMetrics")) + .toList(); static { SearchModule searchModule = new SearchModule(Settings.EMPTY, List.of()); @@ -706,6 +711,9 @@ public class OpenSearchClient implements SearchClient { new os.org.opensearch.action.search.SearchRequest( Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + List sourceFieldsToExcludeCopy = new ArrayList<>(SOURCE_FIELDS_TO_EXCLUDE); + sourceFieldsToExcludeCopy.add("lineage"); + searchSourceBuilder.fetchSource(null, sourceFieldsToExcludeCopy.toArray(String[]::new)); searchSourceBuilder.query( QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn))); searchRequest.source(searchSourceBuilder.size(1000)); @@ -773,6 +781,7 @@ public class OpenSearchClient implements SearchClient { new os.org.opensearch.action.search.SearchRequest( Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new)); searchSourceBuilder.query( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn)))); @@ -791,7 +800,7 @@ public class OpenSearchClient implements SearchClient { List> lineage = (List>) hit.getSourceAsMap().get("lineage"); HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); - tempMap.keySet().removeAll(FIELDS_TO_REMOVE); + tempMap.remove("lineage"); nodes.add(tempMap); for (Map lin : lineage) { HashMap fromEntity = (HashMap) lin.get("fromEntity"); @@ -949,6 +958,7 @@ public class OpenSearchClient implements SearchClient { QueryBuilders.boolQuery() .must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn))); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new)); searchSourceBuilder.query(boolQueryBuilder); if (CommonUtil.nullOrEmpty(deleted)) { searchSourceBuilder.query( @@ -964,7 +974,7 @@ public class OpenSearchClient implements SearchClient { List> lineage = (List>) hit.getSourceAsMap().get("lineage"); HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); - tempMap.keySet().removeAll(FIELDS_TO_REMOVE); + tempMap.remove("lineage"); nodes.add(tempMap); for (Map lin : lineage) { HashMap fromEntity = (HashMap) lin.get("fromEntity");