yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-25 17:08:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(search): support sorting on multiple fields (#10775)

Browse Source
This commit is contained in:
RyanHolstien 2024-08-09 15:29:43 -05:00 committed by GitHub
parent 4d2af40465
commit 06562f320d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
69 changed files with 519 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java
View File

@ -159,9 +159,10 @@ public final class GetChartsResolver implements DataFetcher<List<AnalyticsChartG
.setValue(
String.valueOf(
trailingMonthDateRange.getStart())))))))),
new SortCriterion()
.setField(CORP_USER_STATUS_LAST_MODIFIED_FIELD_NAME)
.setOrder(SortOrder.DESCENDING),
Collections.singletonList(
new SortCriterion()
.setField(CORP_USER_STATUS_LAST_MODIFIED_FIELD_NAME)
.setOrder(SortOrder.DESCENDING)),
0,
100);
}

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java
View File

@ -77,7 +77,8 @@ public final class GetMetadataAnalyticsResolver implements DataFetcher<List<Anal
}
SearchResult searchResult =
_entityClient.searchAcrossEntities(opContext, entities, query, filter, 0, 0, null, null);
_entityClient.searchAcrossEntities(
opContext, entities, query, filter, 0, 0, Collections.emptyList(), null);
List<AggregationMetadata> aggregationMetadataList =
searchResult.getMetadata().getAggregations();

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/DebugAccessResolver.java
View File

@ -33,6 +33,7 @@ import graphql.schema.DataFetchingEnvironment;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -181,7 +182,7 @@ public class DebugAccessResolver implements DataFetcher<CompletableFuture<DebugA
Constants.POLICY_ENTITY_NAME,
"",
buildFilterToGetPolicies(user, groups, roles),
sortCriterion,
Collections.singletonList(sortCriterion),
0,
10000)
.getEntities()

11
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java
View File

@ -59,10 +59,11 @@ public class ListAccessTokensResolver
if (AuthorizationUtils.canManageTokens(context)
|| isListingSelfTokens(filters, context)) {
try {
final SortCriterion sortCriterion =
new SortCriterion()
.setField(EXPIRES_AT_FIELD_NAME)
.setOrder(SortOrder.DESCENDING);
final List<SortCriterion> sortCriteria =
Collections.singletonList(
new SortCriterion()
.setField(EXPIRES_AT_FIELD_NAME)
.setOrder(SortOrder.DESCENDING));
final SearchResult searchResult =
_entityClient.search(
context
@ -74,7 +75,7 @@ public class ListAccessTokensResolver
filters,
Collections.emptyList(),
context.getOperationContext().getAspectRetriever()),
sortCriterion,
sortCriteria,
start,
count);

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/container/ContainerEntitiesResolver.java
View File

@ -19,6 +19,7 @@ import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import lombok.extern.slf4j.Slf4j;
@ -92,7 +93,7 @@ public class ContainerEntitiesResolver implements DataFetcher<CompletableFuture<
new CriterionArray(ImmutableList.of(filterCriterion))))),
start,
count,
null,
Collections.emptyList(),
null));
} catch (Exception e) {

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java
View File

@ -19,6 +19,7 @@ import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.Collections;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
@ -98,7 +99,7 @@ public class DomainEntitiesResolver implements DataFetcher<CompletableFuture<Sea
new ConjunctiveCriterion().setAnd(criteria))),
start,
count,
null,
Collections.emptyList(),
null));
} catch (Exception e) {

8
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java
View File

@ -22,6 +22,7 @@ import com.linkedin.metadata.search.SearchResult;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@ -66,9 +67,10 @@ public class ListDomainsResolver implements DataFetcher<CompletableFuture<ListDo
Constants.DOMAIN_ENTITY_NAME,
query,
filter,
new SortCriterion()
.setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME)
.setOrder(SortOrder.DESCENDING),
Collections.singletonList(
new SortCriterion()
.setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME)
.setOrder(SortOrder.DESCENDING)),
start,
count);

8
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/ListGroupsResolver.java
View File

@ -21,6 +21,7 @@ import com.linkedin.metadata.search.SearchResult;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -64,9 +65,10 @@ public class ListGroupsResolver implements DataFetcher<CompletableFuture<ListGro
CORP_GROUP_ENTITY_NAME,
query,
null,
new SortCriterion()
.setField(CORP_GROUP_CREATED_TIME_INDEX_FIELD_NAME)
.setOrder(SortOrder.DESCENDING),
Collections.singletonList(
new SortCriterion()
.setField(CORP_GROUP_CREATED_TIME_INDEX_FIELD_NAME)
.setOrder(SortOrder.DESCENDING)),
start,
count);

9
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/EntityIncidentsResolver.java
View File

@ -21,6 +21,7 @@ import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -60,13 +61,13 @@ public class EntityIncidentsResolver
// Index!
// We use the search index so that we can easily sort by the last updated time.
final Filter filter = buildIncidentsEntityFilter(entityUrn, maybeState);
final SortCriterion sortCriterion = buildIncidentsSortCriterion();
final List<SortCriterion> sortCriteria = buildIncidentsSortCriteria();
final SearchResult searchResult =
_entityClient.filter(
context.getOperationContext(),
Constants.INCIDENT_ENTITY_NAME,
filter,
sortCriterion,
sortCriteria,
start,
count);
@ -118,10 +119,10 @@ public class EntityIncidentsResolver
return QueryUtils.newFilter(criterionMap);
}
private SortCriterion buildIncidentsSortCriterion() {
private List<SortCriterion> buildIncidentsSortCriteria() {
final SortCriterion sortCriterion = new SortCriterion();
sortCriterion.setField(CREATED_TIME_SEARCH_INDEX_FIELD_NAME);
sortCriterion.setOrder(SortOrder.DESCENDING);
return sortCriterion;
return Collections.singletonList(sortCriterion);
}
}

8
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/IngestionSourceExecutionRequestsResolver.java
View File

@ -23,6 +23,7 @@ import com.linkedin.metadata.search.SearchEntity;
import com.linkedin.metadata.search.SearchResult;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@ -76,9 +77,10 @@ public class IngestionSourceExecutionRequestsResolver
new ConjunctiveCriterion()
.setAnd(
new CriterionArray(ImmutableList.of(filterCriterion))))),
new SortCriterion()
.setField(REQUEST_TIME_MS_FIELD_NAME)
.setOrder(SortOrder.DESCENDING),
Collections.singletonList(
new SortCriterion()
.setField(REQUEST_TIME_MS_FIELD_NAME)
.setOrder(SortOrder.DESCENDING)),
start,
count);

8
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/ListSecretsResolver.java
View File

@ -26,6 +26,7 @@ import com.linkedin.secret.DataHubSecretValue;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -73,9 +74,10 @@ public class ListSecretsResolver implements DataFetcher<CompletableFuture<ListSe
Constants.SECRETS_ENTITY_NAME,
query,
null,
new SortCriterion()
.setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME)
.setOrder(SortOrder.DESCENDING),
Collections.singletonList(
new SortCriterion()
.setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME)
.setOrder(SortOrder.DESCENDING)),
start,
count);

9
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/jobs/DataJobRunsResolver.java
View File

@ -26,6 +26,7 @@ import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -61,13 +62,13 @@ public class DataJobRunsResolver
// Index!
// We use the search index so that we can easily sort by the last updated time.
final Filter filter = buildTaskRunsEntityFilter(entityUrn);
final SortCriterion sortCriterion = buildTaskRunsSortCriterion();
final List<SortCriterion> sortCriteria = buildTaskRunsSortCriteria();
final SearchResult gmsResult =
_entityClient.filter(
context.getOperationContext(),
Constants.DATA_PROCESS_INSTANCE_ENTITY_NAME,
filter,
sortCriterion,
sortCriteria,
start,
count);
final List<Urn> dataProcessInstanceUrns =
@ -123,10 +124,10 @@ public class DataJobRunsResolver
return filter;
}
private SortCriterion buildTaskRunsSortCriterion() {
private List<SortCriterion> buildTaskRunsSortCriteria() {
final SortCriterion sortCriterion = new SortCriterion();
sortCriterion.setField(CREATED_TIME_SEARCH_INDEX_FIELD_NAME);
sortCriterion.setOrder(SortOrder.DESCENDING);
return sortCriterion;
return Collections.singletonList(sortCriterion);
}
}

9
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/jobs/EntityRunsResolver.java
View File

@ -27,6 +27,7 @@ import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -67,13 +68,13 @@ public class EntityRunsResolver
// Index!
// We use the search index so that we can easily sort by the last updated time.
final Filter filter = buildTaskRunsEntityFilter(entityUrn, direction);
final SortCriterion sortCriterion = buildTaskRunsSortCriterion();
final List<SortCriterion> sortCriteria = buildTaskRunsSortCriteria();
final SearchResult gmsResult =
_entityClient.filter(
context.getOperationContext(),
Constants.DATA_PROCESS_INSTANCE_ENTITY_NAME,
filter,
sortCriterion,
sortCriteria,
start,
count);
final List<Urn> dataProcessInstanceUrns =
@ -133,10 +134,10 @@ public class EntityRunsResolver
return filter;
}
private SortCriterion buildTaskRunsSortCriterion() {
private List<SortCriterion> buildTaskRunsSortCriteria() {
final SortCriterion sortCriterion = new SortCriterion();
sortCriterion.setField(CREATED_TIME_SEARCH_INDEX_FIELD_NAME);
sortCriterion.setOrder(SortOrder.DESCENDING);
return sortCriterion;
return Collections.singletonList(sortCriterion);
}
}

2
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java
View File

@ -67,7 +67,7 @@ public class ListOwnershipTypesResolver
filters,
Collections.emptyList(),
context.getOperationContext().getAspectRetriever()),
DEFAULT_SORT_CRITERION,
Collections.singletonList(DEFAULT_SORT_CRITERION),
start,
count);

13
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/post/ListPostsResolver.java
View File

@ -18,7 +18,9 @@ import com.linkedin.metadata.search.SearchEntity;
import com.linkedin.metadata.search.SearchResult;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@ -49,10 +51,11 @@ public class ListPostsResolver implements DataFetcher<CompletableFuture<ListPost
return GraphQLConcurrencyUtils.supplyAsync(
() -> {
try {
final SortCriterion sortCriterion =
new SortCriterion()
.setField(LAST_MODIFIED_FIELD_NAME)
.setOrder(SortOrder.DESCENDING);
final List<SortCriterion> sortCriteria =
Collections.singletonList(
new SortCriterion()
.setField(LAST_MODIFIED_FIELD_NAME)
.setOrder(SortOrder.DESCENDING));
// First, get all Post Urns.
final SearchResult gmsResult =
@ -61,7 +64,7 @@ public class ListPostsResolver implements DataFetcher<CompletableFuture<ListPost
POST_ENTITY_NAME,
query,
null,
sortCriterion,
sortCriteria,
start,
count);

7
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java
View File

@ -61,8 +61,9 @@ public class ListQueriesResolver implements DataFetcher<CompletableFuture<ListQu
return GraphQLConcurrencyUtils.supplyAsync(
() -> {
try {
final SortCriterion sortCriterion =
new SortCriterion().setField(CREATED_AT_FIELD).setOrder(SortOrder.DESCENDING);
final List<SortCriterion> sortCriteria =
Collections.singletonList(
new SortCriterion().setField(CREATED_AT_FIELD).setOrder(SortOrder.DESCENDING));
// First, get all Query Urns.
final SearchResult gmsResult =
@ -74,7 +75,7 @@ public class ListQueriesResolver implements DataFetcher<CompletableFuture<ListQu
QUERY_ENTITY_NAME,
query,
buildFilters(input, context.getOperationContext().getAspectRetriever()),
sortCriterion,
sortCriteria,
start,
count);

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java
View File

@ -22,6 +22,7 @@ import com.linkedin.view.DataHubViewInfo;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@ -94,7 +95,7 @@ public class AggregateAcrossEntitiesResolver
: inputFilter,
0,
0, // 0 entity count because we don't want resolved entities
null,
Collections.emptyList(),
facets));
} catch (Exception e) {
log.error(

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolver.java
View File

@ -25,6 +25,7 @@ import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import io.datahubproject.metadata.context.OperationContext;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
@ -107,7 +108,7 @@ public class GetQuickFiltersResolver
: null,
0,
0,
null,
Collections.emptyList(),
null);
}

26
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java
View File

@ -18,8 +18,10 @@ import com.linkedin.metadata.service.ViewService;
import com.linkedin.view.DataHubViewInfo;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -65,10 +67,24 @@ public class SearchAcrossEntitiesResolver implements DataFetcher<CompletableFutu
context.getOperationContext().getAspectRetriever());
SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags());
SortCriterion sortCriterion =
input.getSortInput() != null
? mapSortCriterion(input.getSortInput().getSortCriterion())
: null;
List<SortCriterion> sortCriteria;
if (input.getSortInput() != null) {
if (input.getSortInput().getSortCriteria() != null) {
sortCriteria =
input.getSortInput().getSortCriteria().stream()
.map(SearchUtils::mapSortCriterion)
.collect(Collectors.toList());
} else {
sortCriteria =
input.getSortInput().getSortCriterion() != null
? Collections.singletonList(
mapSortCriterion(input.getSortInput().getSortCriterion()))
: Collections.emptyList();
}
} else {
sortCriteria = Collections.emptyList();
}
try {
log.debug(
@ -100,7 +116,7 @@ public class SearchAcrossEntitiesResolver implements DataFetcher<CompletableFutu
: baseFilter,
start,
count,
sortCriterion));
sortCriteria));
} catch (Exception e) {
log.error(
"Failed to execute search for multiple entities: entity types {}, query {}, filters: {}, start: {}, count: {}",

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java
View File

@ -20,6 +20,7 @@ import com.linkedin.metadata.query.SearchFlags;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import io.opentelemetry.extension.annotations.WithSpan;
import java.util.Collections;
import java.util.concurrent.CompletableFuture;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -89,7 +90,7 @@ public class SearchResolver implements DataFetcher<CompletableFuture<SearchResul
input.getFilters(),
input.getOrFilters(),
context.getOperationContext().getAspectRetriever()),
null,
Collections.emptyList(),
start,
count));
} catch (Exception e) {

2
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java
View File

@ -74,7 +74,7 @@ public class ListGlobalViewsResolver implements DataFetcher<CompletableFuture<Li
Constants.DATAHUB_VIEW_ENTITY_NAME,
query,
buildFilters(context.getOperationContext().getAspectRetriever()),
DEFAULT_SORT_CRITERION,
Collections.singletonList(DEFAULT_SORT_CRITERION),
start,
count);

2
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java
View File

@ -79,7 +79,7 @@ public class ListMyViewsResolver implements DataFetcher<CompletableFuture<ListVi
viewType,
context.getActorUrn(),
context.getOperationContext().getAspectRetriever()),
DEFAULT_SORT_CRITERION,
Collections.singletonList(DEFAULT_SORT_CRITERION),
start,
count);

7
datahub-graphql-core/src/main/resources/search.graphql
View File

@ -1372,7 +1372,12 @@ input SearchSortInput {
"""
A criterion to sort search results on
"""
sortCriterion: SortCriterion!
sortCriterion: SortCriterion @deprecated(reason: "Use sortCriteria instead")
"""
A list of values to sort search results on
"""
sortCriteria: [SortCriterion!]
}
"""

4
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java
View File

@ -12,11 +12,11 @@ import com.linkedin.datahub.graphql.generated.ListAccessTokenInput;
import com.linkedin.datahub.graphql.generated.ListAccessTokenResult;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.metadata.Constants;
import com.linkedin.metadata.query.filter.SortCriterion;
import com.linkedin.metadata.search.SearchEntityArray;
import com.linkedin.metadata.search.SearchResult;
import graphql.schema.DataFetchingEnvironment;
import java.util.Collections;
import java.util.List;
import org.mockito.Mockito;
import org.testng.annotations.Test;
@ -47,7 +47,7 @@ public class ListAccessTokensResolverTest {
Mockito.eq(Constants.ACCESS_TOKEN_ENTITY_NAME),
Mockito.eq(""),
Mockito.eq(buildFilter(filters, Collections.emptyList(), null)),
Mockito.any(SortCriterion.class),
Mockito.any(List.class),
Mockito.eq(input.getStart()),
Mockito.eq(input.getCount())))
.thenReturn(

2
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/container/ContainerEntitiesResolverTest.java
View File

@ -62,7 +62,7 @@ public class ContainerEntitiesResolverTest {
new CriterionArray(ImmutableList.of(filterCriterion)))))),
Mockito.eq(0),
Mockito.eq(20),
Mockito.eq(null),
Mockito.eq(Collections.emptyList()),
Mockito.eq(null)))
.thenReturn(
new SearchResult()

2
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolverTest.java
View File

@ -68,7 +68,7 @@ public class DomainEntitiesResolverTest {
new CriterionArray(ImmutableList.of(filterCriterion)))))),
Mockito.eq(0),
Mockito.eq(20),
Mockito.eq(null),
Mockito.eq(Collections.emptyList()),
Mockito.eq(null)))
.thenReturn(
new SearchResult()

15
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java
View File

@ -20,6 +20,7 @@ import com.linkedin.metadata.search.SearchEntityArray;
import com.linkedin.metadata.search.SearchResult;
import com.linkedin.r2.RemoteInvocationException;
import graphql.schema.DataFetchingEnvironment;
import java.util.Collections;
import java.util.concurrent.CompletionException;
import org.mockito.Mockito;
import org.testng.annotations.Test;
@ -47,9 +48,10 @@ public class ListDomainsResolverTest {
Mockito.eq(""),
Mockito.eq(DomainUtils.buildParentDomainFilter(TEST_PARENT_DOMAIN_URN)),
Mockito.eq(
new SortCriterion()
.setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME)
.setOrder(SortOrder.DESCENDING)),
Collections.singletonList(
new SortCriterion()
.setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME)
.setOrder(SortOrder.DESCENDING))),
Mockito.eq(0),
Mockito.eq(20)))
.thenReturn(
@ -90,9 +92,10 @@ public class ListDomainsResolverTest {
Mockito.eq(""),
Mockito.eq(DomainUtils.buildParentDomainFilter(null)),
Mockito.eq(
new SortCriterion()
.setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME)
.setOrder(SortOrder.DESCENDING)),
Collections.singletonList(
new SortCriterion()
.setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME)
.setOrder(SortOrder.DESCENDING))),
Mockito.eq(0),
Mockito.eq(20)))
.thenReturn(

3
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/incident/EntityIncidentsResolverTest.java
View File

@ -37,6 +37,7 @@ import com.linkedin.metadata.search.SearchResult;
import com.linkedin.metadata.search.utils.QueryUtils;
import graphql.schema.DataFetchingEnvironment;
import io.datahubproject.metadata.context.OperationContext;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.mockito.Mockito;
@ -92,7 +93,7 @@ public class EntityIncidentsResolverTest {
Mockito.any(),
Mockito.eq(Constants.INCIDENT_ENTITY_NAME),
Mockito.eq(expectedFilter),
Mockito.eq(expectedSort),
Mockito.eq(Collections.singletonList(expectedSort)),
Mockito.eq(0),
Mockito.eq(10)))
.thenReturn(

4
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/IngestionSourceExecutionRequestsResolverTest.java
View File

@ -22,7 +22,6 @@ import com.linkedin.execution.ExecutionRequestInput;
import com.linkedin.execution.ExecutionRequestResult;
import com.linkedin.metadata.Constants;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.query.filter.SortCriterion;
import com.linkedin.metadata.search.SearchEntity;
import com.linkedin.metadata.search.SearchEntityArray;
import com.linkedin.metadata.search.SearchResult;
@ -30,6 +29,7 @@ import com.linkedin.r2.RemoteInvocationException;
import graphql.schema.DataFetchingEnvironment;
import io.datahubproject.metadata.context.OperationContext;
import java.util.HashSet;
import java.util.List;
import org.mockito.Mockito;
import org.testng.annotations.Test;
@ -46,7 +46,7 @@ public class IngestionSourceExecutionRequestsResolverTest {
any(),
Mockito.eq(Constants.EXECUTION_REQUEST_ENTITY_NAME),
Mockito.any(Filter.class),
Mockito.any(SortCriterion.class),
Mockito.any(List.class),
Mockito.eq(0),
Mockito.eq(10)))
.thenReturn(

6
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/ListSecretsResolverTest.java
View File

@ -15,7 +15,6 @@ import com.linkedin.entity.EnvelopedAspect;
import com.linkedin.entity.EnvelopedAspectMap;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.metadata.Constants;
import com.linkedin.metadata.query.filter.SortCriterion;
import com.linkedin.metadata.search.SearchEntity;
import com.linkedin.metadata.search.SearchEntityArray;
import com.linkedin.metadata.search.SearchResult;
@ -24,6 +23,7 @@ import com.linkedin.secret.DataHubSecretValue;
import graphql.schema.DataFetchingEnvironment;
import io.datahubproject.metadata.context.OperationContext;
import java.util.HashSet;
import java.util.List;
import org.mockito.Mockito;
import org.testng.annotations.Test;
@ -44,7 +44,7 @@ public class ListSecretsResolverTest {
Mockito.eq(Constants.SECRETS_ENTITY_NAME),
Mockito.eq(""),
Mockito.eq(null),
Mockito.any(SortCriterion.class),
Mockito.any(List.class),
Mockito.eq(0),
Mockito.eq(20)))
.thenReturn(
@ -112,7 +112,7 @@ public class ListSecretsResolverTest {
Mockito.any(),
Mockito.eq(""),
Mockito.eq(null),
Mockito.any(SortCriterion.class),
Mockito.any(List.class),
Mockito.anyInt(),
Mockito.anyInt());
}

7
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java
View File

@ -69,9 +69,10 @@ public class ListQueriesResolverTest {
: input.getQuery()),
Mockito.eq(buildFilter(input.getSource(), input.getDatasetUrn())),
Mockito.eq(
new SortCriterion()
.setField(ListQueriesResolver.CREATED_AT_FIELD)
.setOrder(SortOrder.DESCENDING)),
Collections.singletonList(
new SortCriterion()
.setField(ListQueriesResolver.CREATED_AT_FIELD)
.setOrder(SortOrder.DESCENDING))),
Mockito.eq(input.getStart()),
Mockito.eq(input.getCount())))
.thenReturn(

6
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java
View File

@ -324,7 +324,7 @@ public class AggregateAcrossEntitiesResolverTest {
Mockito.any(),
Mockito.anyInt(),
Mockito.anyInt(),
Mockito.eq(null),
Mockito.eq(Collections.emptyList()),
Mockito.eq(null)))
.thenThrow(new RemoteInvocationException());
@ -397,7 +397,7 @@ public class AggregateAcrossEntitiesResolverTest {
Mockito.eq(filter),
Mockito.eq(start),
Mockito.eq(limit),
Mockito.eq(null),
Mockito.eq(Collections.emptyList()),
Mockito.eq(facets)))
.thenReturn(result);
return client;
@ -420,7 +420,7 @@ public class AggregateAcrossEntitiesResolverTest {
Mockito.eq(filter),
Mockito.eq(start),
Mockito.eq(limit),
Mockito.eq(null),
Mockito.eq(Collections.emptyList()),
Mockito.eq(facets));
}

5
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolverTest.java
View File

@ -24,6 +24,7 @@ import com.linkedin.metadata.service.ViewService;
import com.linkedin.r2.RemoteInvocationException;
import graphql.schema.DataFetchingEnvironment;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletionException;
import java.util.stream.Collectors;
@ -114,7 +115,7 @@ public class GetQuickFiltersResolverTest {
Mockito.any(),
Mockito.anyInt(),
Mockito.anyInt(),
Mockito.eq(null),
Mockito.eq(Collections.emptyList()),
Mockito.eq(null)))
.thenThrow(new RemoteInvocationException());
@ -300,7 +301,7 @@ public class GetQuickFiltersResolverTest {
Mockito.eq(filter),
Mockito.eq(start),
Mockito.eq(limit),
Mockito.eq(null),
Mockito.eq(Collections.emptyList()),
Mockito.eq(null)))
.thenReturn(result);
return client;

8
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java
View File

@ -437,8 +437,8 @@ public class SearchAcrossEntitiesResolverTest {
Mockito.any(),
Mockito.anyInt(),
Mockito.anyInt(),
Mockito.eq(null),
Mockito.eq(null)))
Mockito.eq(Collections.emptyList()),
Mockito.eq(Collections.emptyList())))
.thenThrow(new RemoteInvocationException());
final SearchAcrossEntitiesResolver resolver =
@ -485,7 +485,7 @@ public class SearchAcrossEntitiesResolverTest {
Mockito.eq(filter),
Mockito.eq(start),
Mockito.eq(limit),
Mockito.eq(null)))
Mockito.eq(Collections.emptyList())))
.thenReturn(result);
return client;
}
@ -506,7 +506,7 @@ public class SearchAcrossEntitiesResolverTest {
Mockito.eq(filter),
Mockito.eq(start),
Mockito.eq(limit),
Mockito.eq(null));
Mockito.eq(Collections.emptyList()));
}
private static void verifyMockViewService(ViewService mockService, Urn viewUrn) {

12
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolverTest.java
View File

@ -18,6 +18,8 @@ import com.linkedin.metadata.search.SearchEntityArray;
import com.linkedin.metadata.search.SearchResult;
import com.linkedin.metadata.search.SearchResultMetadata;
import graphql.schema.DataFetchingEnvironment;
import java.util.Collections;
import java.util.List;
import org.mockito.Mockito;
import org.testng.annotations.Test;
@ -56,7 +58,7 @@ public class SearchResolverTest {
Constants.DATASET_ENTITY_NAME, // Verify that merged entity types were used.
"",
null,
null,
Collections.emptyList(),
0,
10,
setConvertSchemaFieldsToDatasets(
@ -97,7 +99,7 @@ public class SearchResolverTest {
Constants.DATASET_ENTITY_NAME, // Verify that merged entity types were used.
"",
null,
null,
Collections.emptyList(),
1,
11,
setConvertSchemaFieldsToDatasets(
@ -129,7 +131,7 @@ public class SearchResolverTest {
Constants.DATASET_ENTITY_NAME, // Verify that merged entity types were used.
"not a wildcard",
null, // Verify that view filter was used.
null,
Collections.emptyList(),
0,
10,
setConvertSchemaFieldsToDatasets(
@ -170,7 +172,7 @@ public class SearchResolverTest {
String entityName,
String query,
Filter filter,
SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int start,
int limit,
com.linkedin.metadata.query.SearchFlags searchFlags)
@ -181,7 +183,7 @@ public class SearchResolverTest {
Mockito.eq(entityName),
Mockito.eq(query),
Mockito.eq(filter),
Mockito.eq(sortCriterion),
Mockito.eq(sortCriteria),
Mockito.eq(start),
Mockito.eq(limit));
}

4
entity-registry/src/main/java/com/linkedin/metadata/aspect/GraphRetriever.java
View File

@ -20,7 +20,7 @@ public interface GraphRetriever {
* @param destinationEntityFilter
* @param relationshipTypes
* @param relationshipFilter
* @param sortCriterion
* @param sortCriteria
* @param scrollId
* @param count
* @param startTimeMillis
@ -35,7 +35,7 @@ public interface GraphRetriever {
@Nonnull Filter destinationEntityFilter,
@Nonnull List<String> relationshipTypes,
@Nonnull RelationshipFilter relationshipFilter,
@Nonnull List<SortCriterion> sortCriterion,
@Nonnull List<SortCriterion> sortCriteria,
@Nullable String scrollId,
int count,
@Nullable Long startTimeMillis,

8
li-utils/src/main/java/com/datahub/util/RecordUtils.java
View File

@ -78,6 +78,14 @@ public class RecordUtils {
}
}
public static String toJsonString(@Nonnull List<? extends RecordTemplate> recordTemplates) {
StringBuilder json = new StringBuilder();
for (RecordTemplate recordTemplate : recordTemplates) {
json.append(toJsonString(recordTemplate));
}
return json.toString();
}
/**
* Creates a {@link RecordTemplate} object from a serialized JSON string.
*

39
metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java
View File

@ -62,6 +62,7 @@ import java.net.URISyntaxException;
import java.time.Clock;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -375,7 +376,13 @@ public class JavaEntityClient implements EntityClient {
return ValidationUtils.validateSearchResult(
opContext,
entitySearchService.search(
opContext, List.of(entity), input, newFilter(requestFilters), null, start, count),
opContext,
List.of(entity),
input,
newFilter(requestFilters),
Collections.emptyList(),
start,
count),
entityService);
}
@ -406,7 +413,7 @@ public class JavaEntityClient implements EntityClient {
opContext.withSearchFlags(flags -> flags.setFulltext(false)),
entity,
newFilter(requestFilters),
null,
Collections.emptyList(),
start,
count)),
entityService);
@ -417,7 +424,7 @@ public class JavaEntityClient implements EntityClient {
*
* @param input search query
* @param filter search filters
* @param sortCriterion sort criterion
* @param sortCriteria sort criteria
* @param start start offset for search results
* @param count max number of search results requested
* @return Snapshot key
@ -429,14 +436,14 @@ public class JavaEntityClient implements EntityClient {
@Nonnull String entity,
@Nonnull String input,
@Nullable Filter filter,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int start,
int count)
throws RemoteInvocationException {
return ValidationUtils.validateSearchResult(
opContext,
entitySearchService.search(
opContext, List.of(entity), input, filter, sortCriterion, start, count),
opContext, List.of(entity), input, filter, sortCriteria, start, count),
entityService);
}
@ -449,10 +456,10 @@ public class JavaEntityClient implements EntityClient {
@Nullable Filter filter,
int start,
int count,
@Nullable SortCriterion sortCriterion)
List<SortCriterion> sortCriteria)
throws RemoteInvocationException {
return searchAcrossEntities(
opContext, entities, input, filter, start, count, sortCriterion, null);
opContext, entities, input, filter, start, count, sortCriteria, null);
}
/**
@ -464,7 +471,7 @@ public class JavaEntityClient implements EntityClient {
* @param start start offset for search results
* @param count max number of search results requested
* @param facets list of facets we want aggregations for
* @param sortCriterion sorting criterion
* @param sortCriteria sorting criteria
* @return Snapshot key
* @throws RemoteInvocationException when unable to execute request
*/
@ -476,7 +483,7 @@ public class JavaEntityClient implements EntityClient {
@Nullable Filter filter,
int start,
int count,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable List<String> facets)
throws RemoteInvocationException {
@ -487,7 +494,7 @@ public class JavaEntityClient implements EntityClient {
entities,
input,
filter,
sortCriterion,
sortCriteria,
start,
count,
facets),
@ -529,7 +536,7 @@ public class JavaEntityClient implements EntityClient {
@Nullable String input,
@Nullable Integer maxHops,
@Nullable Filter filter,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int start,
int count)
throws RemoteInvocationException {
@ -543,7 +550,7 @@ public class JavaEntityClient implements EntityClient {
input,
maxHops,
filter,
sortCriterion,
sortCriteria,
start,
count),
entityService);
@ -559,7 +566,7 @@ public class JavaEntityClient implements EntityClient {
@Nullable String input,
@Nullable Integer maxHops,
@Nullable Filter filter,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nonnull String keepAlive,
int count)
@ -577,7 +584,7 @@ public class JavaEntityClient implements EntityClient {
input,
maxHops,
filter,
sortCriterion,
sortCriteria,
scrollId,
keepAlive,
count),
@ -645,7 +652,7 @@ public class JavaEntityClient implements EntityClient {
@Nonnull OperationContext opContext,
@Nonnull String entity,
@Nonnull Filter filter,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int start,
int count)
throws RemoteInvocationException {
@ -655,7 +662,7 @@ public class JavaEntityClient implements EntityClient {
opContext.withSearchFlags(flags -> flags.setFulltext(true)),
entity,
filter,
sortCriterion,
sortCriteria,
start,
count),
entityService);

2
metadata-io/src/main/java/com/linkedin/metadata/graph/dgraph/DgraphGraphService.java
View File

@ -788,7 +788,7 @@ public class DgraphGraphService implements GraphService {
@Nonnull Filter destinationEntityFilter,
@Nonnull List<String> relationshipTypes,
@Nonnull RelationshipFilter relationshipFilter,
@Nonnull List<SortCriterion> sortCriterion,
@Nonnull List<SortCriterion> sortCriteria,
@Nullable String scrollId,
int count,
@Nullable Long startTimeMillis,

8
metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java
View File

@ -1323,7 +1323,7 @@ public class ESGraphQueryDAO {
@Nullable final Filter destinationEntityFilter,
@Nonnull final List<String> relationshipTypes,
@Nonnull final RelationshipFilter relationshipFilter,
@Nonnull List<SortCriterion> sortCriterion,
@Nonnull List<SortCriterion> sortCriteria,
@Nullable String scrollId,
int count) {
@ -1336,12 +1336,12 @@ public class ESGraphQueryDAO {
relationshipTypes,
relationshipFilter);
return executeScrollSearchQuery(finalQuery, sortCriterion, scrollId, count);
return executeScrollSearchQuery(finalQuery, sortCriteria, scrollId, count);
}
private SearchResponse executeScrollSearchQuery(
@Nonnull final QueryBuilder query,
@Nonnull List<SortCriterion> sortCriterion,
@Nonnull List<SortCriterion> sortCriteria,
@Nullable String scrollId,
final int count) {
@ -1357,7 +1357,7 @@ public class ESGraphQueryDAO {
searchSourceBuilder.size(count);
searchSourceBuilder.query(query);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, List.of(), false);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriteria, List.of(), false);
searchRequest.source(searchSourceBuilder);
ESUtils.setSearchAfter(searchSourceBuilder, sort, null, null);

4
metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java
View File

@ -315,7 +315,7 @@ public class ElasticSearchGraphService implements GraphService, ElasticSearchInd
@Nullable Filter destinationEntityFilter,
@Nonnull List<String> relationshipTypes,
@Nonnull RelationshipFilter relationshipFilter,
@Nonnull List<SortCriterion> sortCriterion,
@Nonnull List<SortCriterion> sortCriteria,
@Nullable String scrollId,
int count,
@Nullable Long startTimeMillis,
@ -331,7 +331,7 @@ public class ElasticSearchGraphService implements GraphService, ElasticSearchInd
destinationEntityFilter,
relationshipTypes,
relationshipFilter,
sortCriterion,
sortCriteria,
scrollId,
count);

2
metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java
View File

@ -921,7 +921,7 @@ public class Neo4jGraphService implements GraphService {
@Nonnull Filter destinationEntityFilter,
@Nonnull List<String> relationshipTypes,
@Nonnull RelationshipFilter relationshipFilter,
@Nonnull List<SortCriterion> sortCriterion,
@Nonnull List<SortCriterion> sortCriteria,
@Nullable String scrollId,
int count,
@Nullable Long startTimeMillis,

26
metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java
View File

@ -118,7 +118,7 @@ public class LineageSearchService {
* @param maxHops the maximum number of hops away to search for. If null, defaults to 1000
* @param inputFilters the request map with fields and values as filters to be applied to search
* hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param from index to start the search from
* @param size the number of search hits to return
* @return a {@link LineageSearchResult} that contains a list of matched documents and related
@ -134,7 +134,7 @@ public class LineageSearchService {
@Nullable String input,
@Nullable Integer maxHops,
@Nullable Filter inputFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size) {
@ -255,7 +255,7 @@ public class LineageSearchService {
SearchUtils.removeCriteria(
inputFilters, criterion -> criterion.getField().equals(DEGREE_FILTER_INPUT));
if (canDoLightning(lineageRelationships, finalInput, reducedFilters, sortCriterion)) {
if (canDoLightning(lineageRelationships, finalInput, reducedFilters, sortCriteria)) {
codePath = "lightning";
// use lightning approach to return lineage search results
LineageSearchResult lineageSearchResult =
@ -276,7 +276,7 @@ public class LineageSearchService {
lineageRelationships,
finalInput,
reducedFilters,
sortCriterion,
sortCriteria,
from,
size);
if (!lineageSearchResult.getEntities().isEmpty()) {
@ -303,7 +303,7 @@ public class LineageSearchService {
List<LineageRelationship> lineageRelationships,
String input,
Filter inputFilters,
SortCriterion sortCriterion) {
List<SortCriterion> sortCriteria) {
boolean simpleFilters =
inputFilters == null
|| inputFilters.getOr() == null
@ -318,7 +318,7 @@ public class LineageSearchService {
return (lineageRelationships.size() > cacheConfiguration.getLightningThreshold())
&& input.equals("*")
&& simpleFilters
&& sortCriterion == null;
&& CollectionUtils.isEmpty(sortCriteria);
}
@VisibleForTesting
@ -533,7 +533,7 @@ public class LineageSearchService {
List<LineageRelationship> lineageRelationships,
@Nonnull String input,
@Nullable Filter inputFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size) {
@ -566,7 +566,7 @@ public class LineageSearchService {
entitiesToQuery,
input,
finalFilter,
sortCriterion,
sortCriteria,
queryFrom,
querySize),
urnToRelationship);
@ -761,7 +761,7 @@ public class LineageSearchService {
* @param maxHops the maximum number of hops away to search for. If null, defaults to 1000
* @param inputFilters the request map with fields and values as filters to be applied to search
* hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param scrollId opaque scroll identifier to pass to search service
* @param size the number of search hits to return
* @return a {@link LineageSearchResult} that contains a list of matched documents and related
@ -777,7 +777,7 @@ public class LineageSearchService {
@Nullable String input,
@Nullable Integer maxHops,
@Nullable Filter inputFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nonnull String keepAlive,
int size) {
@ -831,7 +831,7 @@ public class LineageSearchService {
lineageRelationships,
input != null ? input : "*",
reducedFilters,
sortCriterion,
sortCriteria,
scrollId,
keepAlive,
size);
@ -843,7 +843,7 @@ public class LineageSearchService {
List<LineageRelationship> lineageRelationships,
@Nonnull String input,
@Nullable Filter inputFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nonnull String keepAlive,
int size) {
@ -878,7 +878,7 @@ public class LineageSearchService {
entitiesToQuery,
input,
finalFilter,
sortCriterion,
sortCriteria,
scrollId,
keepAlive,
querySize),

28
metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java
View File

@ -65,7 +65,7 @@ public class SearchService {
* @param input the search input text
* @param postFilters the request map with fields and values as filters to be applied to search
* hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param from index to start the search from
* @param size the number of search hits to return
* @return a {@link SearchResult} that contains a list of matched documents and related search
@ -77,7 +77,7 @@ public class SearchService {
@Nonnull List<String> entityNames,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size) {
List<String> entitiesToSearch = getEntitiesToSearch(opContext, entityNames, size);
@ -87,7 +87,7 @@ public class SearchService {
}
SearchResult result =
_cachingEntitySearchService.search(
opContext, entitiesToSearch, input, postFilters, sortCriterion, from, size, null);
opContext, entitiesToSearch, input, postFilters, sortCriteria, from, size, null);
try {
return result
@ -105,11 +105,11 @@ public class SearchService {
@Nonnull List<String> entities,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size) {
return searchAcrossEntities(
opContext, entities, input, postFilters, sortCriterion, from, size, null);
opContext, entities, input, postFilters, sortCriteria, from, size, null);
}
/**
@ -120,7 +120,7 @@ public class SearchService {
* @param input the search input text
* @param postFilters the request map with fields and values as filters to be applied to search
* hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param from index to start the search from
* @param size the number of search hits to return
* @param facets list of facets we want aggregations for
@ -133,14 +133,14 @@ public class SearchService {
@Nonnull List<String> entities,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size,
@Nullable List<String> facets) {
log.debug(
String.format(
"Searching Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, from: %s, size: %s",
entities, input, postFilters, sortCriterion, from, size));
entities, input, postFilters, sortCriteria, from, size));
// DEPRECATED
// This is the legacy version of `_entityType`-- it operates as a special case and does not
// support ORs, Unions, etc.
@ -160,7 +160,7 @@ public class SearchService {
}
SearchResult result =
_cachingEntitySearchService.search(
opContext, nonEmptyEntities, input, postFilters, sortCriterion, from, size, facets);
opContext, nonEmptyEntities, input, postFilters, sortCriteria, from, size, facets);
if (facets == null || facets.contains("entity") || facets.contains("_entityType")) {
Optional<AggregationMetadata> entityTypeAgg =
result.getMetadata().getAggregations().stream()
@ -238,7 +238,7 @@ public class SearchService {
* @param input the search input text
* @param postFilters the request map with fields and values as filters to be applied to search
* hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param scrollId opaque scroll identifier for passing to search backend
* @param size the number of search hits to return
* @return a {@link ScrollResult} that contains a list of matched documents and related search
@ -250,21 +250,21 @@ public class SearchService {
@Nonnull List<String> entities,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nullable String keepAlive,
int size) {
log.debug(
String.format(
"Searching Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, from: %s, size: %s",
entities, input, postFilters, sortCriterion, scrollId, size));
"Searching Search documents entities: %s, input: %s, postFilters: %s, sortCriteria: %s, from: %s, size: %s",
entities, input, postFilters, sortCriteria, scrollId, size));
List<String> entitiesToSearch = getEntitiesToSearch(opContext, entities, size);
if (entitiesToSearch.isEmpty()) {
// No indices with non-zero entries: skip querying and return empty result
return getEmptyScrollResult(size);
}
return _cachingEntitySearchService.scroll(
opContext, entitiesToSearch, input, postFilters, sortCriterion, scrollId, keepAlive, size);
opContext, entitiesToSearch, input, postFilters, sortCriteria, scrollId, keepAlive, size);
}
private static SearchResult getEmptySearchResult(int from, int size) {

2
metadata-io/src/main/java/com/linkedin/metadata/search/SearchServiceSearchRetriever.java
View File

@ -43,7 +43,7 @@ public class SearchServiceSearchRetriever implements SearchRetriever {
entities,
"*",
filters,
urnSort,
List.of(urnSort),
scrollId,
null,
count);

23
metadata-io/src/main/java/com/linkedin/metadata/search/cache/CachedSearchResult.java vendored Normal file
View File

@ -0,0 +1,23 @@
package com.linkedin.metadata.search.cache;
import static com.datahub.util.RecordUtils.*;
import static com.linkedin.metadata.search.utils.GZIPUtil.*;
import com.linkedin.metadata.search.SearchResult;
import java.io.Serializable;
import lombok.Data;
@Data
public class CachedSearchResult implements Serializable {
private final byte[] searchResult;
private final long timestamp;
public CachedSearchResult(SearchResult lineageResult, long timestamp) {
this.searchResult = gzipCompress(toJsonString(lineageResult));
this.timestamp = timestamp;
}
public SearchResult getSearchResult() {
return toRecordTemplate(SearchResult.class, gzipDecompress(searchResult));
}
}

37
metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java
View File

@ -20,6 +20,7 @@ import java.util.Optional;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import lombok.RequiredArgsConstructor;
import org.apache.commons.collections.CollectionUtils;
import org.javatuples.Septet;
import org.javatuples.Sextet;
import org.springframework.cache.Cache;
@ -47,7 +48,7 @@ public class CachingEntitySearchService {
* @param entityNames the names of the entity to search
* @param query the search query
* @param filters the filters to include
* @param sortCriterion the sort criterion
* @param sortCriteria the sort criteria
* @param from the start offset
* @param size the count
* @param facets list of facets we want aggregations for
@ -58,12 +59,12 @@ public class CachingEntitySearchService {
@Nonnull List<String> entityNames,
@Nonnull String query,
@Nullable Filter filters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size,
@Nullable List<String> facets) {
return getCachedSearchResults(
opContext, entityNames, query, filters, sortCriterion, from, size, facets);
opContext, entityNames, query, filters, sortCriteria, from, size, facets);
}
/**
@ -115,7 +116,7 @@ public class CachingEntitySearchService {
* @param entities the names of the entities to search
* @param query the search query
* @param filters the filters to include
* @param sortCriterion the sort criterion
* @param sortCriteria the sort criteria
* @param scrollId opaque scroll identifier for a scroll request
* @param keepAlive the string representation of how long to keep point in time alive
* @param size the count
@ -126,12 +127,12 @@ public class CachingEntitySearchService {
@Nonnull List<String> entities,
@Nonnull String query,
@Nullable Filter filters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nullable String keepAlive,
int size) {
return getCachedScrollResults(
opContext, entities, query, filters, sortCriterion, scrollId, keepAlive, size);
opContext, entities, query, filters, sortCriteria, scrollId, keepAlive, size);
}
/**
@ -145,7 +146,7 @@ public class CachingEntitySearchService {
@Nonnull List<String> entityNames,
@Nonnull String query,
@Nullable Filter filters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size,
@Nullable List<String> facets) {
@ -158,7 +159,7 @@ public class CachingEntitySearchService {
entityNames,
query,
filters,
sortCriterion,
sortCriteria,
querySize.getFrom(),
querySize.getSize(),
facets),
@ -168,7 +169,7 @@ public class CachingEntitySearchService {
entityNames,
query,
filters != null ? toJsonString(filters) : null,
sortCriterion != null ? toJsonString(sortCriterion) : null,
CollectionUtils.isNotEmpty(sortCriteria) ? toJsonString(sortCriteria) : null,
facets,
querySize),
enableCache)
@ -269,7 +270,7 @@ public class CachingEntitySearchService {
@Nonnull List<String> entities,
@Nonnull String query,
@Nullable Filter filters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nullable String keepAlive,
int size) {
@ -291,7 +292,7 @@ public class CachingEntitySearchService {
entities,
query,
filters != null ? toJsonString(filters) : null,
sortCriterion != null ? toJsonString(sortCriterion) : null,
CollectionUtils.isNotEmpty(sortCriteria) ? toJsonString(sortCriteria) : null,
scrollId,
size);
String json = cache.get(cacheKey, String.class);
@ -305,7 +306,7 @@ public class CachingEntitySearchService {
entities,
query,
filters,
sortCriterion,
sortCriteria,
scrollId,
keepAlive,
size,
@ -321,7 +322,7 @@ public class CachingEntitySearchService {
entities,
query,
filters,
sortCriterion,
sortCriteria,
scrollId,
keepAlive,
size,
@ -337,12 +338,12 @@ public class CachingEntitySearchService {
final List<String> entityNames,
final String input,
final Filter filters,
final SortCriterion sortCriterion,
final List<SortCriterion> sortCriteria,
final int start,
final int count,
@Nullable final List<String> facets) {
return entitySearchService.search(
opContext, entityNames, input, filters, sortCriterion, start, count, facets);
opContext, entityNames, input, filters, sortCriteria, start, count, facets);
}
/** Executes the expensive autocomplete query using the {@link EntitySearchService} */
@ -373,17 +374,17 @@ public class CachingEntitySearchService {
final List<String> entities,
final String input,
final Filter filters,
final SortCriterion sortCriterion,
final List<SortCriterion> sortCriteria,
@Nullable final String scrollId,
@Nullable final String keepAlive,
final int count,
final boolean fulltext) {
if (fulltext) {
return entitySearchService.fullTextScroll(
opContext, entities, input, filters, sortCriterion, scrollId, keepAlive, count);
opContext, entities, input, filters, sortCriteria, scrollId, keepAlive, count);
} else {
return entitySearchService.structuredScroll(
opContext, entities, input, filters, sortCriterion, scrollId, keepAlive, count);
opContext, entities, input, filters, sortCriteria, scrollId, keepAlive, count);
}
}

40
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
View File

@ -142,10 +142,10 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI
@Nonnull List<String> entityNames,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size) {
return search(opContext, entityNames, input, postFilters, sortCriterion, from, size, null);
return search(opContext, entityNames, input, postFilters, sortCriteria, from, size, null);
}
@Nonnull
@ -154,14 +154,14 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI
@Nonnull List<String> entityNames,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size,
@Nullable List<String> facets) {
log.debug(
String.format(
"Searching FullText Search documents entityName: %s, input: %s, postFilters: %s, sortCriterion: %s, from: %s, size: %s",
entityNames, input, postFilters, sortCriterion, from, size));
"Searching FullText Search documents entityName: %s, input: %s, postFilters: %s, sortCriteria: %s, from: %s, size: %s",
entityNames, input, postFilters, sortCriteria, from, size));
return esSearchDAO.search(
opContext.withSearchFlags(
@ -169,7 +169,7 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI
entityNames,
input,
postFilters,
sortCriterion,
sortCriteria,
from,
size,
facets);
@ -181,20 +181,20 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI
@Nonnull OperationContext opContext,
@Nonnull String entityName,
@Nullable Filter filters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size) {
log.debug(
String.format(
"Filtering Search documents entityName: %s, filters: %s, sortCriterion: %s, from: %s, size: %s",
entityName, filters, sortCriterion, from, size));
"Filtering Search documents entityName: %s, filters: %s, sortCriteria: %s, from: %s, size: %s",
entityName, filters, sortCriteria, from, size));
return esSearchDAO.filter(
opContext.withSearchFlags(
flags -> applyDefaultSearchFlags(flags, null, DEFAULT_SERVICE_SEARCH_FLAGS)),
entityName,
filters,
sortCriterion,
sortCriteria,
from,
size);
}
@ -330,14 +330,14 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI
@Nonnull List<String> entities,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nullable String keepAlive,
int size) {
log.debug(
String.format(
"Scrolling Structured Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, scrollId: %s, size: %s",
entities, input, postFilters, sortCriterion, scrollId, size));
"Scrolling Structured Search documents entities: %s, input: %s, postFilters: %s, sortCriteria: %s, scrollId: %s, size: %s",
entities, input, postFilters, sortCriteria, scrollId, size));
return esSearchDAO.scroll(
opContext.withSearchFlags(
@ -347,7 +347,7 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI
entities,
input,
postFilters,
sortCriterion,
sortCriteria,
scrollId,
keepAlive,
size);
@ -360,14 +360,14 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI
@Nonnull List<String> entities,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nullable String keepAlive,
int size) {
log.debug(
String.format(
"Scrolling FullText Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, scrollId: %s, size: %s",
entities, input, postFilters, sortCriterion, scrollId, size));
"Scrolling FullText Search documents entities: %s, input: %s, postFilters: %s, sortCriteria: %s, scrollId: %s, size: %s",
entities, input, postFilters, sortCriteria, scrollId, size));
return esSearchDAO.scroll(
opContext.withSearchFlags(
@ -377,7 +377,7 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI
entities,
input,
postFilters,
sortCriterion,
sortCriteria,
scrollId,
keepAlive,
size);
@ -400,7 +400,7 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI
@Nonnull String documentId,
@Nonnull String entityName,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nullable String keepAlive,
int size,
@ -413,7 +413,7 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI
documentId,
entityName,
postFilters,
sortCriterion,
sortCriteria,
scrollId,
keepAlive,
size,

34
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java
View File

@ -228,7 +228,7 @@ public class ESSearchDAO {
* @param input the search input text
* @param postFilters the request map with fields and values as filters to be applied to search
* hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param from index to start the search from
* @param size the number of search hits to return
* @param facets list of facets we want aggregations for
@ -241,7 +241,7 @@ public class ESSearchDAO {
@Nonnull List<String> entityNames,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size,
@Nullable List<String> facets) {
@ -257,7 +257,7 @@ public class ESSearchDAO {
final SearchRequest searchRequest =
SearchRequestHandler.getBuilder(entitySpecs, searchConfiguration, customSearchConfiguration)
.getSearchRequest(
opContext, finalInput, transformedFilters, sortCriterion, from, size, facets);
opContext, finalInput, transformedFilters, sortCriteria, from, size, facets);
searchRequest.indices(
entityNames.stream().map(indexConvention::getEntityIndexName).toArray(String[]::new));
searchRequestTimer.stop();
@ -270,7 +270,7 @@ public class ESSearchDAO {
*
* @param filters the request map with fields and values to be applied as filters to the search
* query
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param from index to start the search from
* @param size number of search hits to return
* @return a {@link SearchResult} that contains a list of filtered documents and related search
@ -281,7 +281,7 @@ public class ESSearchDAO {
@Nonnull OperationContext opContext,
@Nonnull String entityName,
@Nullable Filter filters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size) {
IndexConvention indexConvention = opContext.getSearchContext().getIndexConvention();
@ -289,7 +289,7 @@ public class ESSearchDAO {
Filter transformedFilters = transformFilterForEntities(filters, indexConvention);
final SearchRequest searchRequest =
SearchRequestHandler.getBuilder(entitySpec, searchConfiguration, customSearchConfiguration)
.getFilterRequest(opContext, transformedFilters, sortCriterion, from, size);
.getFilterRequest(opContext, transformedFilters, sortCriteria, from, size);
searchRequest.indices(indexConvention.getIndexName(entitySpec));
return executeAndExtract(
@ -401,7 +401,7 @@ public class ESSearchDAO {
* @param input the search input text
* @param postFilters the request map with fields and values as filters to be applied to search
* hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param scrollId opaque scroll Id to convert to a PIT ID and Sort array to pass to ElasticSearch
* @param keepAlive string representation of the time to keep a point in time alive
* @param size the number of search hits to return
@ -414,7 +414,7 @@ public class ESSearchDAO {
@Nonnull List<String> entities,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nullable String keepAlive,
int size) {
@ -439,7 +439,7 @@ public class ESSearchDAO {
transformedFilters,
entitySpecs,
finalInput,
sortCriterion,
sortCriteria,
null);
// PIT specifies indices in creation so it doesn't support specifying indices on the request, so
@ -462,7 +462,7 @@ public class ESSearchDAO {
@Nullable Filter postFilters,
List<EntitySpec> entitySpecs,
String finalInput,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable List<String> facets) {
String pitId = null;
Object[] sort = null;
@ -483,15 +483,7 @@ public class ESSearchDAO {
return SearchRequestHandler.getBuilder(
entitySpecs, searchConfiguration, customSearchConfiguration)
.getSearchRequest(
opContext,
finalInput,
postFilters,
sortCriterion,
sort,
pitId,
keepAlive,
size,
facets);
opContext, finalInput, postFilters, sortCriteria, sort, pitId, keepAlive, size, facets);
}
public Optional<SearchResponse> raw(
@ -544,7 +536,7 @@ public class ESSearchDAO {
@Nonnull String documentId,
@Nonnull String entityName,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nullable String keepAlive,
int size,
@ -564,7 +556,7 @@ public class ESSearchDAO {
transformedFilters,
Collections.singletonList(entitySpec),
finalQuery,
sortCriterion,
sortCriteria,
facets);
;

14
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
View File

@ -187,7 +187,7 @@ public class SearchRequestHandler {
@Nonnull OperationContext opContext,
@Nonnull String input,
@Nullable Filter filter,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size,
@Nullable List<String> facets) {
@ -213,7 +213,7 @@ public class SearchRequestHandler {
if (Boolean.FALSE.equals(searchFlags.isSkipHighlighting())) {
searchSourceBuilder.highlighter(highlights);
}
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, entitySpecs);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriteria, entitySpecs);
if (Boolean.TRUE.equals(searchFlags.isGetSuggestions())) {
ESUtils.buildNameSuggestions(searchSourceBuilder, input);
@ -243,7 +243,7 @@ public class SearchRequestHandler {
@Nonnull OperationContext opContext,
@Nonnull String input,
@Nullable Filter filter,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable Object[] sort,
@Nullable String pitId,
@Nullable String keepAlive,
@ -272,7 +272,7 @@ public class SearchRequestHandler {
if (Boolean.FALSE.equals(searchFlags.isSkipHighlighting())) {
searchSourceBuilder.highlighter(highlights);
}
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, entitySpecs);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriteria, entitySpecs);
searchRequest.source(searchSourceBuilder);
log.debug("Search request is: " + searchRequest);
searchRequest.indicesOptions(null);
@ -285,7 +285,7 @@ public class SearchRequestHandler {
* to be applied to search results.
*
* @param filters {@link Filter} list of conditions with fields and values
* @param sortCriterion {@link SortCriterion} to be applied to the search results
* @param sortCriteria list of {@link SortCriterion} to be applied to the search results
* @param from index to start the search from
* @param size the number of search hits to return
* @return {@link SearchRequest} that contains the filtered query
@ -294,7 +294,7 @@ public class SearchRequestHandler {
public SearchRequest getFilterRequest(
@Nonnull OperationContext opContext,
@Nullable Filter filters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size) {
SearchRequest searchRequest = new SearchRequest();
@ -303,7 +303,7 @@ public class SearchRequestHandler {
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(filterQuery);
searchSourceBuilder.from(from).size(size);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, entitySpecs);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriteria, entitySpecs);
searchRequest.source(searchSourceBuilder);
return searchRequest;

29
metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
View File

@ -299,17 +299,14 @@ public class ESUtils {
*
* @param searchSourceBuilder {@link SearchSourceBuilder} that needs to be populated with sort
* order
* @param sortCriterion {@link SortCriterion} to be applied to the search results
* @param sortCriteria list of {@link SortCriterion} to be applied to the search results
*/
public static void buildSortOrder(
@Nonnull SearchSourceBuilder searchSourceBuilder,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
List<EntitySpec> entitySpecs) {
buildSortOrder(
searchSourceBuilder,
sortCriterion == null ? List.of() : List.of(sortCriterion),
entitySpecs,
true);
searchSourceBuilder, sortCriteria == null ? List.of() : sortCriteria, entitySpecs, true);
}
/**
@ -321,20 +318,20 @@ public class ESUtils {
*/
public static void buildSortOrder(
@Nonnull SearchSourceBuilder searchSourceBuilder,
@Nonnull List<SortCriterion> sortCriterion,
@Nonnull List<SortCriterion> sortCriteria,
List<EntitySpec> entitySpecs,
boolean enableDefaultSort) {
if (sortCriterion.isEmpty() && enableDefaultSort) {
if (sortCriteria.isEmpty() && enableDefaultSort) {
searchSourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC));
} else {
for (SortCriterion sortCriteria : sortCriterion) {
for (SortCriterion sortCriterion : sortCriteria) {
Optional<SearchableAnnotation.FieldType> fieldTypeForDefault = Optional.empty();
for (EntitySpec entitySpec : entitySpecs) {
List<SearchableFieldSpec> fieldSpecs = entitySpec.getSearchableFieldSpecs();
for (SearchableFieldSpec fieldSpec : fieldSpecs) {
SearchableAnnotation annotation = fieldSpec.getSearchableAnnotation();
if (annotation.getFieldName().equals(sortCriteria.getField())
|| annotation.getFieldNameAliases().contains(sortCriteria.getField())) {
if (annotation.getFieldName().equals(sortCriterion.getField())
|| annotation.getFieldNameAliases().contains(sortCriterion.getField())) {
fieldTypeForDefault = Optional.of(fieldSpec.getSearchableAnnotation().getFieldType());
break;
}
@ -346,15 +343,15 @@ public class ESUtils {
if (fieldTypeForDefault.isEmpty() && !entitySpecs.isEmpty()) {
log.warn(
"Sort criterion field "
+ sortCriteria.getField()
+ sortCriterion.getField()
+ " was not found in any entity spec to be searched");
}
final SortOrder esSortOrder =
(sortCriteria.getOrder() == com.linkedin.metadata.query.filter.SortOrder.ASCENDING)
(sortCriterion.getOrder() == com.linkedin.metadata.query.filter.SortOrder.ASCENDING)
? SortOrder.ASC
: SortOrder.DESC;
FieldSortBuilder sortBuilder =
new FieldSortBuilder(sortCriteria.getField()).order(esSortOrder);
new FieldSortBuilder(sortCriterion.getField()).order(esSortOrder);
if (fieldTypeForDefault.isPresent()) {
String esFieldtype = getElasticTypeForFieldType(fieldTypeForDefault.get());
if (esFieldtype != null) {
@ -365,8 +362,8 @@ public class ESUtils {
}
}
if (enableDefaultSort
&& (sortCriterion.isEmpty()
|| sortCriterion.stream()
&& (sortCriteria.isEmpty()
|| sortCriteria.stream()
.noneMatch(c -> c.getField().equals(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD)))) {
searchSourceBuilder.sort(
new FieldSortBuilder(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD).order(SortOrder.ASC));

8
metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java
View File

@ -551,7 +551,7 @@ public class ElasticSearchTimeseriesAspectService
@Nonnull String entityName,
@Nonnull String aspectName,
@Nullable Filter filter,
@Nonnull List<SortCriterion> sortCriterion,
@Nonnull List<SortCriterion> sortCriteria,
@Nullable String scrollId,
int count,
@Nullable Long startTimeMillis,
@ -592,7 +592,7 @@ public class ElasticSearchTimeseriesAspectService
entityName,
aspectName,
filterQueryBuilder,
sortCriterion,
sortCriteria,
scrollId,
count);
int totalCount = (int) response.getHits().getTotalHits().value;
@ -615,7 +615,7 @@ public class ElasticSearchTimeseriesAspectService
@Nonnull final String entityName,
@Nonnull final String aspectName,
@Nonnull final QueryBuilder query,
@Nonnull List<SortCriterion> sortCriterion,
@Nonnull List<SortCriterion> sortCriteria,
@Nullable String scrollId,
final int count) {
@ -631,7 +631,7 @@ public class ElasticSearchTimeseriesAspectService
searchSourceBuilder.size(count);
searchSourceBuilder.query(query);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, List.of(), false);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriteria, List.of(), false);
searchRequest.source(searchSourceBuilder);
ESUtils.setSearchAfter(searchSourceBuilder, sort, null, null);

2
metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java
View File

@ -2196,7 +2196,7 @@ public abstract class GraphServiceTestBase extends AbstractTestNGSpringContextTe
relationships.stream()
.flatMap(relationship -> relationship.getDegrees().stream())
.reduce(0, Math::max);
assertTrue(maxDegree > 1);
assertTrue(maxDegree >= 1);
EntityLineageResult lineageResultMulti =
getGraphService(true)

6
metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java
View File

@ -216,7 +216,7 @@ public abstract class SampleDataFixtureTestBase extends AbstractTestNGSpringCont
SearchSourceBuilder builder = new SearchSourceBuilder();
SortCriterion sortCriterion =
new SortCriterion().setOrder(SortOrder.DESCENDING).setField(dateFieldName);
ESUtils.buildSortOrder(builder, sortCriterion, entitySpecs);
ESUtils.buildSortOrder(builder, Collections.singletonList(sortCriterion), entitySpecs);
List<SortBuilder<?>> sorts = builder.sorts();
assertEquals(sorts.size(), 2); // sort by last modified and then by urn
for (SortBuilder sort : sorts) {
@ -235,7 +235,7 @@ public abstract class SampleDataFixtureTestBase extends AbstractTestNGSpringCont
SearchSourceBuilder nameBuilder = new SearchSourceBuilder();
SortCriterion nameCriterion =
new SortCriterion().setOrder(SortOrder.ASCENDING).setField(entityNameField);
ESUtils.buildSortOrder(nameBuilder, nameCriterion, entitySpecs);
ESUtils.buildSortOrder(nameBuilder, Collections.singletonList(nameCriterion), entitySpecs);
sorts = nameBuilder.sorts();
assertEquals(sorts.size(), 2);
for (SortBuilder sort : sorts) {
@ -1959,7 +1959,7 @@ public abstract class SampleDataFixtureTestBase extends AbstractTestNGSpringCont
SEARCHABLE_ENTITIES,
query,
null,
criterion,
Collections.singletonList(criterion),
0,
100,
null);

21
metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java
View File

@ -61,6 +61,7 @@ import io.datahubproject.openapi.util.OpenApiEntitiesUtil;
import io.datahubproject.openapi.v1.entities.EntitiesController;
import jakarta.servlet.http.HttpServletRequest;
import java.net.URISyntaxException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -628,12 +629,18 @@ public class EntityApiDelegateImpl<I, O, S> {
authentication,
true);
// TODO multi-field sort
SortCriterion sortCriterion = new SortCriterion();
sortCriterion.setField(Optional.ofNullable(sort).map(s -> s.get(0)).orElse("urn"));
sortCriterion.setOrder(
com.linkedin.metadata.query.filter.SortOrder.valueOf(
Optional.ofNullable(sortOrder).map(Enum::name).orElse("ASCENDING")));
List<SortCriterion> sortCriteria =
Optional.ofNullable(sort).orElse(Collections.singletonList("urn")).stream()
.map(
sortField -> {
SortCriterion sortCriterion = new SortCriterion();
sortCriterion.setField(sortField);
sortCriterion.setOrder(
com.linkedin.metadata.query.filter.SortOrder.valueOf(
Optional.ofNullable(sortOrder).map(Enum::name).orElse("ASCENDING")));
return sortCriterion;
})
.collect(Collectors.toList());
ScrollResult result =
_searchService.scrollAcrossEntities(
@ -641,7 +648,7 @@ public class EntityApiDelegateImpl<I, O, S> {
List.of(entitySpec.getName()),
query,
null,
sortCriterion,
sortCriteria,
scrollId,
null,
count);

15
metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java
View File

@ -61,6 +61,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.util.CollectionUtils;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PatchMapping;
@ -166,6 +167,7 @@ public abstract class GenericEntitiesController<
@RequestParam(value = "query", defaultValue = "*") String query,
@RequestParam(value = "scrollId", required = false) String scrollId,
@RequestParam(value = "sort", required = false, defaultValue = "urn") String sortField,
@RequestParam(value = "sortCriteria", required = false) List<String> sortFields,
@RequestParam(value = "sortOrder", required = false, defaultValue = "ASCENDING")
String sortOrder,
@RequestParam(value = "systemMetadata", required = false, defaultValue = "false")
@ -194,8 +196,15 @@ public abstract class GenericEntitiesController<
authentication,
true);
// TODO: support additional and multiple sort params
SortCriterion sortCriterion = SearchUtil.sortBy(sortField, SortOrder.valueOf(sortOrder));
List<SortCriterion> sortCriteria;
if (!CollectionUtils.isEmpty(sortFields)) {
sortCriteria = new ArrayList<>();
sortFields.forEach(
field -> sortCriteria.add(SearchUtil.sortBy(field, SortOrder.valueOf(sortOrder))));
} else {
sortCriteria =
Collections.singletonList(SearchUtil.sortBy(sortField, SortOrder.valueOf(sortOrder)));
}
ScrollResult result =
searchService.scrollAcrossEntities(
@ -206,7 +215,7 @@ public abstract class GenericEntitiesController<
List.of(entitySpec.getName()),
query,
null,
sortCriterion,
sortCriteria,
scrollId,
null,
count);

21
metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java
View File

@ -20,6 +20,7 @@ import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs;
import com.linkedin.metadata.entity.restoreindices.RestoreIndicesResult;
import com.linkedin.metadata.query.SearchFlags;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.query.filter.SortCriterion;
import com.linkedin.metadata.search.EntitySearchService;
import com.linkedin.metadata.systemmetadata.SystemMetadataService;
import com.linkedin.metadata.timeseries.TimeseriesAspectService;
@ -214,6 +215,13 @@ public class OperationsController {
@RequestParam(value = "filters", required = false)
@Nullable
String filters,
@Parameter(
name = "sortCriteria",
required = false,
description = "Criteria to sort results on.")
@RequestParam("sortCriteria")
@Nullable
List<SortCriterion> sortCriteria,
@Parameter(name = "searchFlags", description = "Optional configuration flags.")
@RequestParam(value = "searchFlags", required = false)
@Nullable
@ -253,7 +261,7 @@ public class OperationsController {
encodeValue(documentId),
entityName,
filters == null ? null : objectMapper.readValue(filters, Filter.class),
null,
sortCriteria,
scrollId,
keepAlive,
size,
@ -315,6 +323,13 @@ public class OperationsController {
@RequestParam(value = "filters", required = false)
@Nullable
String filters,
@Parameter(
name = "sortCriteria",
required = false,
description = "Criteria to sort results on.")
@RequestParam("sortCriteria")
@Nullable
List<SortCriterion> sortCriteria,
@Parameter(name = "searchFlags", description = "Optional configuration flags.")
@RequestParam(value = "searchFlags", required = false)
@Nullable
@ -354,7 +369,7 @@ public class OperationsController {
encodeValue(documentIdA),
entityName,
filters == null ? null : objectMapper.readValue(filters, Filter.class),
null,
sortCriteria,
scrollId,
keepAlive,
size,
@ -367,7 +382,7 @@ public class OperationsController {
encodeValue(documentIdB),
entityName,
filters == null ? null : objectMapper.readValue(filters, Filter.class),
null,
sortCriteria,
scrollId,
keepAlive,
size,

4
metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimeseriesController.java
View File

@ -90,7 +90,7 @@ public class TimeseriesController {
throw new IllegalArgumentException("Only timeseries aspects are supported.");
}
List<SortCriterion> sortCriterion =
List<SortCriterion> sortCriteria =
List.of(
SearchUtil.sortBy("timestampMillis", SortOrder.DESCENDING),
SearchUtil.sortBy("messageId", SortOrder.DESCENDING));
@ -101,7 +101,7 @@ public class TimeseriesController {
entityName,
aspectName,
null,
sortCriterion,
sortCriteria,
scrollId,
count,
startTimeMillis,

7
metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/controller/EntityControllerTest.java
View File

@ -37,6 +37,7 @@ import com.linkedin.metadata.utils.SearchUtil;
import io.datahubproject.metadata.context.OperationContext;
import io.datahubproject.openapi.config.SpringWebConfig;
import io.datahubproject.test.metadata.context.TestOperationContexts;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
@ -95,7 +96,7 @@ public class EntityControllerTest extends AbstractTestNGSpringContextTests {
eq(List.of("dataset")),
anyString(),
nullable(Filter.class),
eq(SearchUtil.sortBy("urn", SortOrder.valueOf("ASCENDING"))),
eq(Collections.singletonList(SearchUtil.sortBy("urn", SortOrder.valueOf("ASCENDING")))),
nullable(String.class),
nullable(String.class),
anyInt()))
@ -113,7 +114,9 @@ public class EntityControllerTest extends AbstractTestNGSpringContextTests {
eq(List.of("dataset")),
anyString(),
nullable(Filter.class),
eq(SearchUtil.sortBy("urn", SortOrder.valueOf("DESCENDING"))),
eq(
Collections.singletonList(
SearchUtil.sortBy("urn", SortOrder.valueOf("DESCENDING")))),
nullable(String.class),
nullable(String.class),
anyInt()))

28
metadata-service/restli-api/src/main/idl/com.linkedin.entity.entities.restspec.json
View File

@ -199,6 +199,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "start",
"type" : "int"
@ -248,6 +252,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "start",
"type" : "int"
@ -288,6 +296,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "scrollId",
"type" : "string",
@ -333,6 +345,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "scrollId",
"type" : "string",
@ -374,6 +390,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "start",
"type" : "int"
@ -411,6 +431,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "start",
"type" : "int"
@ -452,6 +476,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "start",
"type" : "int"

28
metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json
View File

@ -6820,6 +6820,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "start",
"type" : "int"
@ -6869,6 +6873,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "start",
"type" : "int"
@ -6909,6 +6917,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "scrollId",
"type" : "string",
@ -6954,6 +6966,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "scrollId",
"type" : "string",
@ -6995,6 +7011,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "start",
"type" : "int"
@ -7032,6 +7052,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "start",
"type" : "int"
@ -7073,6 +7097,10 @@
"name" : "sort",
"type" : "com.linkedin.metadata.query.filter.SortCriterion",
"optional" : true
}, {
"name" : "sortCriteria",
"type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.metadata.query.filter.SortCriterion\" }",
"optional" : true
}, {
"name" : "start",
"type" : "int"

20
metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java
View File

@ -235,7 +235,7 @@ public interface EntityClient {
*
* @param input search query
* @param filter search filters
* @param sortCriterion sort criterion
* @param sortCriteria sort criteria
* @param start start offset for search results
* @param count max number of search results requested
* @return Snapshot key
@ -246,7 +246,7 @@ public interface EntityClient {
@Nonnull String entity,
@Nonnull String input,
@Nullable Filter filter,
SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int start,
int count)
throws RemoteInvocationException;
@ -270,7 +270,7 @@ public interface EntityClient {
@Nullable Filter filter,
int start,
int count,
@Nullable SortCriterion sortCriterion)
List<SortCriterion> sortCriteria)
throws RemoteInvocationException;
/**
@ -292,7 +292,7 @@ public interface EntityClient {
@Nullable Filter filter,
int start,
int count,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
List<String> facets)
throws RemoteInvocationException;
@ -328,7 +328,7 @@ public interface EntityClient {
* @param input the search input text
* @param maxHops the max number of hops away to search for. If null, searches all hops.
* @param filter the request map with fields and values as filters to be applied to search hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param start index to start the search from
* @param count the number of search hits to return
* @return a {@link SearchResult} that contains a list of matched documents and related search
@ -342,7 +342,7 @@ public interface EntityClient {
@Nonnull String input,
@Nullable Integer maxHops,
@Nullable Filter filter,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int start,
int count)
throws RemoteInvocationException;
@ -356,7 +356,7 @@ public interface EntityClient {
* @param input the search input text
* @param maxHops the max number of hops away to search for. If null, searches all hops.
* @param filter the request map with fields and values as filters to be applied to search hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param scrollId opaque scroll ID indicating offset
* @param keepAlive string representation of time to keep point in time alive, ex: 5m
* @param count the number of search hits to return of roundtrips for UI visualizations.
@ -372,7 +372,7 @@ public interface EntityClient {
@Nonnull String input,
@Nullable Integer maxHops,
@Nullable Filter filter,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nonnull String keepAlive,
int count)
@ -426,7 +426,7 @@ public interface EntityClient {
*
* @param entity filter entity
* @param filter search filters
* @param sortCriterion sort criterion
* @param sortCriteria sort criteria
* @param start start offset for search results
* @param count max number of search results requested
* @return a set of {@link SearchResult}s
@ -436,7 +436,7 @@ public interface EntityClient {
@Nonnull OperationContext opContext,
@Nonnull String entity,
@Nonnull Filter filter,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int start,
int count)
throws RemoteInvocationException;

45
metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java
View File

@ -63,6 +63,7 @@ import com.linkedin.metadata.query.filter.Criterion;
import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.query.filter.SortCriterion;
import com.linkedin.metadata.query.filter.SortCriterionArray;
import com.linkedin.metadata.search.LineageScrollResult;
import com.linkedin.metadata.search.LineageSearchResult;
import com.linkedin.metadata.search.ScrollResult;
@ -101,6 +102,7 @@ import javax.mail.MethodNotSupportedException;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.NotImplementedException;
import org.opensearch.core.common.util.CollectionUtils;
@Slf4j
public class RestliEntityClient extends BaseClient implements EntityClient {
@ -592,7 +594,7 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
*
* @param input search query
* @param filter search filters
* @param sortCriterion sort criterion
* @param sortCriteria sort criteria
* @param start start offset for search results
* @param count max number of search results requested
* @return Snapshot key
@ -605,7 +607,7 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
@Nonnull String entity,
@Nonnull String input,
@Nullable Filter filter,
SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int start,
int count)
throws RemoteInvocationException {
@ -623,8 +625,9 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
requestBuilder.filterParam(filter);
}
if (sortCriterion != null) {
requestBuilder.sortParam(sortCriterion);
if (!CollectionUtils.isEmpty(sortCriteria)) {
requestBuilder.sortParam(sortCriteria.get(0));
requestBuilder.sortCriteriaParam(new SortCriterionArray(sortCriteria));
}
if (searchFlags != null) {
@ -646,10 +649,10 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
@Nullable Filter filter,
int start,
int count,
@Nullable SortCriterion sortCriterion)
List<SortCriterion> sortCriteria)
throws RemoteInvocationException {
return searchAcrossEntities(
opContext, entities, input, filter, start, count, sortCriterion, null);
opContext, entities, input, filter, start, count, sortCriteria, null);
}
/**
@ -673,7 +676,7 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
@Nullable Filter filter,
int start,
int count,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable List<String> facets)
throws RemoteInvocationException {
@ -695,8 +698,9 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
requestBuilder.searchFlagsParam(searchFlags);
}
if (sortCriterion != null) {
requestBuilder.sortParam(sortCriterion);
if (!CollectionUtils.isEmpty(sortCriteria)) {
requestBuilder.sortParam(sortCriteria.get(0));
requestBuilder.sortCriteriaParam(new SortCriterionArray(sortCriteria));
}
return sendClientRequest(requestBuilder, opContext.getAuthentication()).getEntity();
@ -746,7 +750,7 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
@Nonnull String input,
@Nullable Integer maxHops,
@Nullable Filter filter,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int start,
int count)
throws RemoteInvocationException {
@ -773,6 +777,12 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
if (lineageFlags.getEndTimeMillis() != null) {
requestBuilder.endTimeMillisParam(lineageFlags.getEndTimeMillis());
}
if (!CollectionUtils.isEmpty(sortCriteria)) {
requestBuilder.sortParam(sortCriteria.get(0));
requestBuilder.sortCriteriaParam(new SortCriterionArray(sortCriteria));
}
requestBuilder.searchFlagsParam(opContext.getSearchContext().getSearchFlags());
return sendClientRequest(requestBuilder, opContext.getAuthentication()).getEntity();
@ -788,7 +798,7 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
@Nonnull String input,
@Nullable Integer maxHops,
@Nullable Filter filter,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nonnull String keepAlive,
int count)
@ -818,6 +828,12 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
if (lineageFlags.getEndTimeMillis() != null) {
requestBuilder.endTimeMillisParam(lineageFlags.getEndTimeMillis());
}
if (!CollectionUtils.isEmpty(sortCriteria)) {
requestBuilder.sortParam(sortCriteria.get(0));
requestBuilder.sortCriteriaParam(new SortCriterionArray(sortCriteria));
}
requestBuilder.searchFlagsParam(opContext.getSearchContext().getSearchFlags());
return sendClientRequest(requestBuilder, opContext.getAuthentication()).getEntity();
@ -906,7 +922,7 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
@Nonnull OperationContext opContext,
@Nonnull String entity,
@Nonnull Filter filter,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int start,
int count)
throws RemoteInvocationException {
@ -917,8 +933,9 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
.filterParam(filter)
.startParam(start)
.countParam(count);
if (sortCriterion != null) {
requestBuilder.sortParam(sortCriterion);
if (!CollectionUtils.isEmpty(sortCriteria)) {
requestBuilder.sortParam(sortCriteria.get(0));
requestBuilder.sortCriteriaParam(new SortCriterionArray(sortCriteria));
}
return sendClientRequest(requestBuilder, opContext.getAuthentication()).getEntity();
}

3
metadata-service/restli-servlet-impl/build.gradle
View File

@ -106,3 +106,6 @@ pegasus.main.idlOptions.addIdlItem([
])
ext.apiProject = project(':metadata-service:restli-api')
spotlessJava.dependsOn generateTestDataTemplate
spotlessJava.dependsOn generateIntegTestDataTemplate

47
metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java
View File

@ -367,6 +367,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
@ActionParam(PARAM_INPUT) @Nonnull String input,
@ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter,
@ActionParam(PARAM_SORT) @Optional @Nullable SortCriterion sortCriterion,
@ActionParam(PARAM_SORT_CRITERIA) @Optional @Nullable SortCriterion[] sortCriteria,
@ActionParam(PARAM_START) int start,
@ActionParam(PARAM_COUNT) int count,
@Optional @Deprecated @Nullable @ActionParam(PARAM_FULLTEXT) Boolean fulltext,
@ -386,6 +387,8 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_SEARCH, entityName), authorizer, auth, true)
.withSearchFlags(flags -> searchFlags != null ? searchFlags : new SearchFlags().setFulltext(Boolean.TRUE.equals(fulltext)));
List<SortCriterion> sortCriterionList = getSortCriteria(sortCriteria, sortCriterion);
log.info("GET SEARCH RESULTS for {} with query {}", entityName, input);
// TODO - change it to use _searchService once we are confident on it's latency
return RestliUtil.toTask(
@ -394,7 +397,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
// This API is not used by the frontend for search bars so we default to structured
result =
entitySearchService.search(opContext,
List.of(entityName), input, filter, sortCriterion, start, count);
List.of(entityName), input, filter, sortCriterionList, start, count);
if (!isAPIAuthorizedResult(
auth,
@ -417,6 +420,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
@ActionParam(PARAM_INPUT) @Nonnull String input,
@ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter,
@ActionParam(PARAM_SORT) @Optional @Nullable SortCriterion sortCriterion,
@ActionParam(PARAM_SORT_CRITERIA) @Optional @Nullable SortCriterion[] sortCriteria,
@ActionParam(PARAM_START) int start,
@ActionParam(PARAM_COUNT) int count,
@ActionParam(PARAM_SEARCH_FLAGS) @Optional SearchFlags searchFlags) {
@ -436,10 +440,12 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
HttpStatus.S_403_FORBIDDEN, "User is unauthorized to search.");
}
List<SortCriterion> sortCriterionList = getSortCriteria(sortCriteria, sortCriterion);
log.info("GET SEARCH RESULTS ACROSS ENTITIES for {} with query {}", entityList, input);
return RestliUtil.toTask(
() -> {
SearchResult result = searchService.searchAcrossEntities(opContext, entityList, input, filter, sortCriterion, start, count);
SearchResult result = searchService.searchAcrossEntities(opContext, entityList, input, filter, sortCriterionList, start, count);
if (!isAPIAuthorizedResult(
auth,
authorizer,
@ -452,6 +458,18 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
});
}
private List<SortCriterion> getSortCriteria(@Nullable SortCriterion[] sortCriteria, @Nullable SortCriterion sortCriterion) {
List<SortCriterion> sortCriterionList;
if (sortCriteria != null) {
sortCriterionList = Arrays.asList(sortCriteria);
} else if (sortCriterion != null) {
sortCriterionList = Collections.singletonList(sortCriterion);
} else {
sortCriterionList = Collections.emptyList();
}
return sortCriterionList;
}
@Action(name = ACTION_SCROLL_ACROSS_ENTITIES)
@Nonnull
@WithSpan
@ -460,6 +478,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
@ActionParam(PARAM_INPUT) @Nonnull String input,
@ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter,
@ActionParam(PARAM_SORT) @Optional @Nullable SortCriterion sortCriterion,
@ActionParam(PARAM_SORT_CRITERIA) @Optional @Nullable SortCriterion[] sortCriteria,
@ActionParam(PARAM_SCROLL_ID) @Optional @Nullable String scrollId,
@ActionParam(PARAM_KEEP_ALIVE) String keepAlive,
@ActionParam(PARAM_COUNT) int count,
@ -479,6 +498,8 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
HttpStatus.S_403_FORBIDDEN, "User is unauthorized to search.");
}
List<SortCriterion> sortCriterionList = getSortCriteria(sortCriteria, sortCriterion);
log.info(
"GET SCROLL RESULTS ACROSS ENTITIES for {} with query {} and scroll ID: {}",
entityList,
@ -492,7 +513,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
entityList,
input,
filter,
sortCriterion,
sortCriterionList,
scrollId,
keepAlive,
count);
@ -520,6 +541,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
@ActionParam(PARAM_MAX_HOPS) @Optional @Nullable Integer maxHops,
@ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter,
@ActionParam(PARAM_SORT) @Optional @Nullable SortCriterion sortCriterion,
@ActionParam(PARAM_SORT_CRITERIA) @Optional @Nullable SortCriterion[] sortCriteria,
@ActionParam(PARAM_START) int start,
@ActionParam(PARAM_COUNT) int count,
@ActionParam(PARAM_START_TIME_MILLIS) @Optional @Nullable Long startTimeMillis,
@ -535,6 +557,8 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
HttpStatus.S_403_FORBIDDEN, "User is unauthorized to search.");
}
List<SortCriterion> sortCriterionList = getSortCriteria(sortCriteria, sortCriterion);
OperationContext opContext = OperationContext.asSession(
systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_SEARCH_ACROSS_LINEAGE, entities), authorizer, auth, true)
.withSearchFlags(flags -> (searchFlags != null ? searchFlags : new SearchFlags().setFulltext(true))
@ -559,7 +583,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
input,
maxHops,
filter,
sortCriterion,
sortCriterionList,
start,
count),
entityService),
@ -577,6 +601,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
@ActionParam(PARAM_MAX_HOPS) @Optional @Nullable Integer maxHops,
@ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter,
@ActionParam(PARAM_SORT) @Optional @Nullable SortCriterion sortCriterion,
@ActionParam(PARAM_SORT_CRITERIA) @Optional @Nullable SortCriterion[] sortCriteria,
@ActionParam(PARAM_SCROLL_ID) @Optional @Nullable String scrollId,
@ActionParam(PARAM_KEEP_ALIVE) String keepAlive,
@ActionParam(PARAM_COUNT) int count,
@ -611,6 +636,8 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
entityList,
input);
List<SortCriterion> sortCriterionList = getSortCriteria(sortCriteria, sortCriterion);
return RestliUtil.toTask(
() ->
validateLineageScrollResult(opContext,
@ -622,7 +649,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
input,
maxHops,
filter,
sortCriterion,
sortCriterionList,
scrollId,
keepAlive,
count),
@ -637,6 +664,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
@ActionParam(PARAM_ENTITY) @Nonnull String entityName,
@ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter,
@ActionParam(PARAM_SORT) @Optional @Nullable SortCriterion sortCriterion,
@ActionParam(PARAM_SORT_CRITERIA) @Optional @Nullable SortCriterion[] sortCriteria,
@ActionParam(PARAM_START) int start,
@ActionParam(PARAM_COUNT) int count) {
@ -653,10 +681,12 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_LIST, entityName), authorizer, auth, true)
.withSearchFlags(flags -> new SearchFlags().setFulltext(false));
List<SortCriterion> sortCriterionList = getSortCriteria(sortCriteria, sortCriterion);
log.info("GET LIST RESULTS for {} with filter {}", entityName, filter);
return RestliUtil.toTask(
() -> {
SearchResult result = entitySearchService.filter(opContext, entityName, filter, sortCriterion, start, count);
SearchResult result = entitySearchService.filter(opContext, entityName, filter, sortCriterionList, start, count);
if (!AuthUtil.isAPIAuthorizedResult(
auth,
authorizer,
@ -1148,6 +1178,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
@ActionParam(PARAM_ENTITY) @Nonnull String entityName,
@ActionParam(PARAM_FILTER) Filter filter,
@ActionParam(PARAM_SORT) @Optional @Nullable SortCriterion sortCriterion,
@ActionParam(PARAM_SORT_CRITERIA) @Optional @Nullable SortCriterion[] sortCriteria,
@ActionParam(PARAM_START) int start,
@ActionParam(PARAM_COUNT) int count) {
@ -1161,10 +1192,12 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
}
OperationContext opContext = OperationContext.asSession(
systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_FILTER, entityName), authorizer, auth, true);
List<SortCriterion> sortCriterionList = getSortCriteria(sortCriteria, sortCriterion);
log.info("FILTER RESULTS for {} with filter {}", entityName, filter);
return RestliUtil.toTask(
() -> {
SearchResult result = entitySearchService.filter(opContext.withSearchFlags(flags -> flags.setFulltext(true)), entityName, filter, sortCriterion, start, count);
SearchResult result = entitySearchService.filter(opContext.withSearchFlags(flags -> flags.setFulltext(true)), entityName, filter, sortCriterionList, start, count);
if (!isAPIAuthorizedResult(
auth,
authorizer,

1
metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java
View File

@ -24,6 +24,7 @@ public final class RestliConstants {
public static final String PARAM_FILTER = "filter";
public static final String PARAM_GROUP = "group";
public static final String PARAM_SORT = "sort";
public static final String PARAM_SORT_CRITERIA = "sortCriteria";
public static final String PARAM_QUERY = "query";
public static final String PARAM_FIELD = "field";
public static final String PARAM_PATH = "path";

2
metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java
View File

@ -137,7 +137,7 @@ public class MockTimeseriesAspectService implements TimeseriesAspectService {
@Nonnull String entityName,
@Nonnull String aspectName,
@Nullable Filter filter,
@Nonnull List<SortCriterion> sortCriterion,
@Nonnull List<SortCriterion> sortCriteria,
@Nullable String scrollId,
int count,
@Nullable Long startTimeMillis,

22
metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
View File

@ -80,7 +80,7 @@ public interface EntitySearchService {
* @param input the search input text
* @param postFilters the request map with fields and values as filters to be applied to search
* hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param from index to start the search from
* @param size the number of search hits to return
* @return a {@link SearchResult} that contains a list of matched documents and related search
@ -92,7 +92,7 @@ public interface EntitySearchService {
@Nonnull List<String> entityNames,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size);
@ -108,7 +108,7 @@ public interface EntitySearchService {
* @param input the search input text
* @param postFilters the request map with fields and values as filters to be applied to search
* hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param from index to start the search from
* @param size the number of search hits to return
* @param facets list of facets we want aggregations for
@ -121,7 +121,7 @@ public interface EntitySearchService {
@Nonnull List<String> entityNames,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size,
@Nullable List<String> facets);
@ -132,7 +132,7 @@ public interface EntitySearchService {
* @param entityName name of the entity
* @param filters the request map with fields and values to be applied as filters to the search
* query
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param from index to start the search from
* @param size number of search hits to return
* @return a {@link SearchResult} that contains a list of filtered documents and related search
@ -143,7 +143,7 @@ public interface EntitySearchService {
@Nonnull OperationContext opContext,
@Nonnull String entityName,
@Nullable Filter filters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
int from,
int size);
@ -265,7 +265,7 @@ public interface EntitySearchService {
* @param input the search input text
* @param postFilters the request map with fields and values as filters to be applied to search
* hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param scrollId opaque scroll identifier to pass to search service
* @param size the number of search hits to return
* @return a {@link ScrollResult} that contains a list of matched documents and related search
@ -277,7 +277,7 @@ public interface EntitySearchService {
@Nonnull List<String> entities,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nullable String keepAlive,
int size);
@ -290,7 +290,7 @@ public interface EntitySearchService {
* @param input the search input text
* @param postFilters the request map with fields and values as filters to be applied to search
* hits
* @param sortCriterion {@link SortCriterion} to be applied to search results
* @param sortCriteria list of {@link SortCriterion} to be applied to search results
* @param scrollId opaque scroll identifier to pass to search service
* @param size the number of search hits to return
* @return a {@link ScrollResult} that contains a list of matched documents and related search
@ -302,7 +302,7 @@ public interface EntitySearchService {
@Nonnull List<String> entities,
@Nonnull String input,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nullable String keepAlive,
int size);
@ -316,7 +316,7 @@ public interface EntitySearchService {
@Nonnull String documentId,
@Nonnull String entityName,
@Nullable Filter postFilters,
@Nullable SortCriterion sortCriterion,
List<SortCriterion> sortCriteria,
@Nullable String scrollId,
@Nullable String keepAlive,
int size,

2
metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java
View File

@ -226,7 +226,7 @@ public interface TimeseriesAspectService {
@Nonnull final String entityName,
@Nonnull final String aspectName,
@Nullable Filter filter,
@Nonnull List<SortCriterion> sortCriterion,
@Nonnull List<SortCriterion> sortCriteria,
@Nullable String scrollId,
int count,
@Nullable Long startTimeMillis,