yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-01 21:23:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR - Reduce lineage response size (#18080)

Browse Source 
* fix import issue

* fix: reduce response size and exclude fields in search client source

---------

Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com>
This commit is contained in:
Teddy 2024-10-03 08:13:13 +02:00 committed by GitHub
parent ebb7fed616
commit 751bceb763
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 24 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -227,6 +227,11 @@ public class ElasticSearchClient implements SearchClient {
"fqnParts", "fqnParts",
"chart_suggest", "chart_suggest",
"field_suggest"); "field_suggest");
private static final List<String> SOURCE_FIELDS_TO_EXCLUDE =
Stream.concat(
FIELDS_TO_REMOVE.stream(),
Stream.of("schemaDefinition", "testSuite", "customMetrics"))
.toList();
static { static {
SearchModule searchModule = new SearchModule(Settings.EMPTY, false, List.of()); SearchModule searchModule = new SearchModule(Settings.EMPTY, false, List.of());
@ -705,13 +710,15 @@ public class ElasticSearchClient implements SearchClient {
new es.org.elasticsearch.action.search.SearchRequest( new es.org.elasticsearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
List<String> sourceFieldsToExcludeCopy = new ArrayList<>(SOURCE_FIELDS_TO_EXCLUDE);
sourceFieldsToExcludeCopy.add("lineage");
searchSourceBuilder.fetchSource(null, sourceFieldsToExcludeCopy.toArray(String[]::new));
searchSourceBuilder.query( searchSourceBuilder.query(
QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn))); QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn)));
searchRequest.source(searchSourceBuilder.size(1000)); searchRequest.source(searchSourceBuilder.size(1000));
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
for (var hit : searchResponse.getHits().getHits()) { for (var hit : searchResponse.getHits().getHits()) {
Map<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); Map<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.keySet().removeAll(FIELDS_TO_REMOVE);
responseMap.put("entity", tempMap); responseMap.put("entity", tempMap);
} }
getLineage( getLineage(
@ -772,6 +779,7 @@ public class ElasticSearchClient implements SearchClient {
new es.org.elasticsearch.action.search.SearchRequest( new es.org.elasticsearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new));
searchSourceBuilder.query( searchSourceBuilder.query(
QueryBuilders.boolQuery() QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn)))); .must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn))));
@ -788,7 +796,7 @@ public class ElasticSearchClient implements SearchClient {
List<Map<String, Object>> lineage = List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage"); (List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.keySet().removeAll(FIELDS_TO_REMOVE); tempMap.remove("lineage");
nodes.add(tempMap); nodes.add(tempMap);
for (Map<String, Object> lin : lineage) { for (Map<String, Object> lin : lineage) {
Map<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity"); Map<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");
@ -963,6 +971,7 @@ public class ElasticSearchClient implements SearchClient {
QueryBuilders.boolQuery() QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn))); .must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn)));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new));
searchSourceBuilder.query(boolQueryBuilder); searchSourceBuilder.query(boolQueryBuilder);
if (CommonUtil.nullOrEmpty(deleted)) { if (CommonUtil.nullOrEmpty(deleted)) {
searchSourceBuilder.query( searchSourceBuilder.query(
@ -977,7 +986,7 @@ public class ElasticSearchClient implements SearchClient {
List<Map<String, Object>> lineage = List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage"); (List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.keySet().removeAll(FIELDS_TO_REMOVE); tempMap.remove("lineage");
nodes.add(tempMap); nodes.add(tempMap);
for (Map<String, Object> lin : lineage) { for (Map<String, Object> lin : lineage) {
HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity"); HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");

14
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -221,6 +221,11 @@ public class OpenSearchClient implements SearchClient {
"fqnParts", "fqnParts",
"chart_suggest", "chart_suggest",
"field_suggest"); "field_suggest");
private static final List<String> SOURCE_FIELDS_TO_EXCLUDE =
Stream.concat(
FIELDS_TO_REMOVE.stream(),
Stream.of("schemaDefinition", "testSuite", "customMetrics"))
.toList();
static { static {
SearchModule searchModule = new SearchModule(Settings.EMPTY, List.of()); SearchModule searchModule = new SearchModule(Settings.EMPTY, List.of());
@ -706,6 +711,9 @@ public class OpenSearchClient implements SearchClient {
new os.org.opensearch.action.search.SearchRequest( new os.org.opensearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
List<String> sourceFieldsToExcludeCopy = new ArrayList<>(SOURCE_FIELDS_TO_EXCLUDE);
sourceFieldsToExcludeCopy.add("lineage");
searchSourceBuilder.fetchSource(null, sourceFieldsToExcludeCopy.toArray(String[]::new));
searchSourceBuilder.query( searchSourceBuilder.query(
QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn))); QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn)));
searchRequest.source(searchSourceBuilder.size(1000)); searchRequest.source(searchSourceBuilder.size(1000));
@ -773,6 +781,7 @@ public class OpenSearchClient implements SearchClient {
new os.org.opensearch.action.search.SearchRequest( new os.org.opensearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new));
searchSourceBuilder.query( searchSourceBuilder.query(
QueryBuilders.boolQuery() QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn)))); .must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn))));
@ -791,7 +800,7 @@ public class OpenSearchClient implements SearchClient {
List<Map<String, Object>> lineage = List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage"); (List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.keySet().removeAll(FIELDS_TO_REMOVE); tempMap.remove("lineage");
nodes.add(tempMap); nodes.add(tempMap);
for (Map<String, Object> lin : lineage) { for (Map<String, Object> lin : lineage) {
HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity"); HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");
@ -949,6 +958,7 @@ public class OpenSearchClient implements SearchClient {
QueryBuilders.boolQuery() QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn))); .must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn)));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new));
searchSourceBuilder.query(boolQueryBuilder); searchSourceBuilder.query(boolQueryBuilder);
if (CommonUtil.nullOrEmpty(deleted)) { if (CommonUtil.nullOrEmpty(deleted)) {
searchSourceBuilder.query( searchSourceBuilder.query(
@ -964,7 +974,7 @@ public class OpenSearchClient implements SearchClient {
List<Map<String, Object>> lineage = List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage"); (List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.keySet().removeAll(FIELDS_TO_REMOVE); tempMap.remove("lineage");
nodes.add(tempMap); nodes.add(tempMap);
for (Map<String, Object> lin : lineage) { for (Map<String, Object> lin : lineage) {
HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity"); HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");