yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-04 14:43:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: Implement pagination in pipeline lineage render (#19295)

Browse Source
This commit is contained in:
Mayur Singal 2025-01-09 15:21:04 +05:30 committed by GitHub
parent 4cad5762ad
commit 8f27ed0455
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 132 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -1257,16 +1257,26 @@ public class ElasticSearchClient implements SearchClient {
Set<String> visitedFQN = new HashSet<>(); Set<String> visitedFQN = new HashSet<>();
Set<Map<String, Object>> edges = new HashSet<>(); Set<Map<String, Object>> edges = new HashSet<>();
Set<Map<String, Object>> nodes = new HashSet<>(); Set<Map<String, Object>> nodes = new HashSet<>();
Object[] searchAfter = null;
long processedRecords = 0;
long totalRecords = -1;
while (totalRecords != processedRecords) {
es.org.elasticsearch.action.search.SearchRequest searchRequest = es.org.elasticsearch.action.search.SearchRequest searchRequest =
new es.org.elasticsearch.action.search.SearchRequest( new es.org.elasticsearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
es.org.elasticsearch.index.query.BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); es.org.elasticsearch.index.query.BoolQueryBuilder boolQueryBuilder =
QueryBuilders.boolQuery();
boolQueryBuilder.should( boolQueryBuilder.should(
QueryBuilders.boolQuery() QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn))); .must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn)));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new)); searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new));
FieldSortBuilder sortBuilder = SortBuilders.fieldSort("fullyQualifiedName");
searchSourceBuilder.sort(sortBuilder);
searchSourceBuilder.query(boolQueryBuilder); searchSourceBuilder.query(boolQueryBuilder);
if (searchAfter != null) {
searchSourceBuilder.searchAfter(searchAfter);
}
if (CommonUtil.nullOrEmpty(deleted)) { if (CommonUtil.nullOrEmpty(deleted)) {
searchSourceBuilder.query( searchSourceBuilder.query(
QueryBuilders.boolQuery() QueryBuilders.boolQuery()
@ -1276,6 +1286,7 @@ public class ElasticSearchClient implements SearchClient {
buildSearchSourceFilter(queryFilter, searchSourceBuilder); buildSearchSourceFilter(queryFilter, searchSourceBuilder);
searchRequest.source(searchSourceBuilder); searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
for (var hit : searchResponse.getHits().getHits()) { for (var hit : searchResponse.getHits().getHits()) {
List<Map<String, Object>> lineage = List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage"); (List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
@ -1308,6 +1319,15 @@ public class ElasticSearchClient implements SearchClient {
} }
} }
} }
totalRecords = searchResponse.getHits().getTotalHits().value;
int currentHits = searchResponse.getHits().getHits().length;
processedRecords += currentHits;
if (currentHits > 0) {
searchAfter = searchResponse.getHits().getHits()[currentHits - 1].getSortValues();
} else {
searchAfter = null;
}
}
getLineage( getLineage(
fqn, fqn,
visitedFQN, visitedFQN,

18
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -1256,6 +1256,10 @@ public class OpenSearchClient implements SearchClient {
Set<Map<String, Object>> edges = new HashSet<>(); Set<Map<String, Object>> edges = new HashSet<>();
Set<Map<String, Object>> nodes = new HashSet<>(); Set<Map<String, Object>> nodes = new HashSet<>();
responseMap.put("entity", null); responseMap.put("entity", null);
Object[] searchAfter = null;
long processedRecords = 0;
long totalRecords = -1;
while (totalRecords != processedRecords) {
os.org.opensearch.action.search.SearchRequest searchRequest = os.org.opensearch.action.search.SearchRequest searchRequest =
new os.org.opensearch.action.search.SearchRequest( new os.org.opensearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
@ -1265,7 +1269,12 @@ public class OpenSearchClient implements SearchClient {
.must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn))); .must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn)));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new)); searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new));
FieldSortBuilder sortBuilder = SortBuilders.fieldSort("fullyQualifiedName");
searchSourceBuilder.sort(sortBuilder);
searchSourceBuilder.query(boolQueryBuilder); searchSourceBuilder.query(boolQueryBuilder);
if (searchAfter != null) {
searchSourceBuilder.searchAfter(searchAfter);
}
if (CommonUtil.nullOrEmpty(deleted)) { if (CommonUtil.nullOrEmpty(deleted)) {
searchSourceBuilder.query( searchSourceBuilder.query(
QueryBuilders.boolQuery() QueryBuilders.boolQuery()
@ -1308,6 +1317,15 @@ public class OpenSearchClient implements SearchClient {
} }
} }
} }
totalRecords = searchResponse.getHits().getTotalHits().value;
int currentHits = searchResponse.getHits().getHits().length;
processedRecords += currentHits;
if (currentHits > 0) {
searchAfter = searchResponse.getHits().getHits()[currentHits - 1].getSortValues();
} else {
searchAfter = null;
}
}
getLineage( getLineage(
fqn, fqn,
visitedFQN, visitedFQN,