yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-26 09:26:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(advanced-search): Complete Advanced Search: backend changes & tying UI together (#6068)

Browse Source 
* stashing progress

* adding remove option

* more progress

* editing

* further in

* additional rendering improvements

* stashing adv search progress

* stashing more progress

* propagating not filters back to UI

* more frontend progress

* more filters working

* getting ready for data platform selector

* add platform select functionality

* locking out switching btwn advanced and standard filters

* final polish

* remove unneeded code

* added unit and cypress tests

* resolutions after merge

* adding documentation

* cleaning up & refactoring

* removing console.log

* minor ui fix & removing unneeded code

* fixing lineage search

* fixing lints

* fix display of degree

* fixing test

* fixing lint

* responding to comments

* fixing tests

* fix smoke tests

* fixing cypress

* fixing cypress test

* responding to comments
This commit is contained in:
Gabe Lyons 2022-10-04 10:20:04 -07:00 committed by GitHub
parent 396fd31ddc
commit ce90310dd0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
64 changed files with 1138 additions and 356 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java
View File

@ -3,10 +3,13 @@ package com.linkedin.datahub.graphql.resolvers;
import com.datahub.authentication.Authentication;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableSet;
import com.linkedin.data.template.StringArray;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.exception.ValidationException;
import com.linkedin.datahub.graphql.generated.FacetFilterInput;
import com.linkedin.datahub.graphql.generated.OrFilter;
import com.linkedin.metadata.query.filter.Condition;
import com.linkedin.metadata.query.filter.Criterion;
import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter;
@ -81,20 +84,77 @@ public class ResolverUtils {
if (!validFacetFields.contains(facetFilterInput.getField())) {
throw new ValidationException(String.format("Unrecognized facet with name %s provided", facetFilterInput.getField()));
}
facetFilters.put(facetFilterInput.getField(), facetFilterInput.getValue());
if (!facetFilterInput.getValues().isEmpty()) {
facetFilters.put(facetFilterInput.getField(), facetFilterInput.getValues().get(0));
}
});
return facetFilters;
}
public static List<Criterion> criterionListFromAndFilter(List<FacetFilterInput> andFilters) {
return andFilters != null && !andFilters.isEmpty()
? andFilters.stream()
.map(filter -> criterionFromFilter(filter))
.collect(Collectors.toList()) : Collections.emptyList();
}
// In the case that user sends filters to be or-d together, we need to build a series of conjunctive criterion
// arrays, rather than just one for the AND case.
public static ConjunctiveCriterionArray buildConjunctiveCriterionArrayWithOr(
@Nonnull List<OrFilter> orFilters
) {
return new ConjunctiveCriterionArray(orFilters.stream().map(orFilter -> {
CriterionArray andCriterionForOr = new CriterionArray(criterionListFromAndFilter(orFilter.getAnd()));
return new ConjunctiveCriterion().setAnd(
andCriterionForOr
);
}
).collect(Collectors.toList()));
}
@Nullable
public static Filter buildFilter(@Nullable List<FacetFilterInput> facetFilterInputs) {
if (facetFilterInputs == null || facetFilterInputs.isEmpty()) {
public static Filter buildFilter(@Nullable List<FacetFilterInput> andFilters, @Nullable List<OrFilter> orFilters) {
if ((andFilters == null || andFilters.isEmpty()) && (orFilters == null || orFilters.isEmpty())) {
return null;
}
return new Filter().setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(facetFilterInputs.stream()
.map(filter -> new Criterion().setField(getFilterField(filter.getField())).setValue(filter.getValue()))
.collect(Collectors.toList())))));
// Or filters are the new default. We will check them first.
// If we have OR filters, we need to build a series of CriterionArrays
if (orFilters != null && !orFilters.isEmpty()) {
return new Filter().setOr(buildConjunctiveCriterionArrayWithOr(orFilters));
}
// If or filters are not set, someone may be using the legacy and filters
final List<Criterion> andCriterions = criterionListFromAndFilter(andFilters);
return new Filter().setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(andCriterions))));
}
// Translates a FacetFilterInput (graphql input class) into Criterion (our internal model)
public static Criterion criterionFromFilter(final FacetFilterInput filter) {
Criterion result = new Criterion();
result.setField(getFilterField(filter.getField()));
if (filter.getValues() != null) {
result.setValues(new StringArray(filter.getValues()));
if (!filter.getValues().isEmpty()) {
result.setValue(filter.getValues().get(0));
} else {
result.setValue("");
}
}
if (filter.getCondition() != null) {
result.setCondition(Condition.valueOf(filter.getCondition().toString()));
} else {
result.setCondition(Condition.EQUAL);
}
if (filter.getNegated() != null) {
result.setNegated(filter.getNegated());
}
return result;
}
private static String getFilterField(final String originalField) {

9
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java
View File

@ -9,14 +9,12 @@ import com.linkedin.datahub.graphql.generated.AssertionRunEventsResult;
import com.linkedin.datahub.graphql.generated.AssertionRunStatus;
import com.linkedin.datahub.graphql.generated.FacetFilterInput;
import com.linkedin.datahub.graphql.generated.FilterInput;
import com.linkedin.datahub.graphql.generated.SearchCondition;
import com.linkedin.datahub.graphql.types.dataset.mappers.AssertionRunEventMapper;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.metadata.Constants;
import com.linkedin.metadata.aspect.EnvelopedAspect;
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
import com.linkedin.metadata.query.filter.Criterion;
import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.r2.RemoteInvocationException;
@ -102,13 +100,16 @@ public class AssertionRunEventResolver implements DataFetcher<CompletableFuture<
}
List<FacetFilterInput> facetFilters = new ArrayList<>();
if (status != null) {
facetFilters.add(new FacetFilterInput("status", status, ImmutableList.of(status), false, SearchCondition.EQUAL));
FacetFilterInput filter = new FacetFilterInput();
filter.setField("status");
filter.setValues(ImmutableList.of(status));
facetFilters.add(filter);
}
if (filtersInput != null) {
facetFilters.addAll(filtersInput.getAnd());
}
return new Filter().setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(facetFilters.stream()
.map(filter -> new Criterion().setField(filter.getField()).setValue(filter.getValue()))
.map(filter -> criterionFromFilter(filter))
.collect(Collectors.toList())))));
}
}

5
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java
View File

@ -1,5 +1,6 @@
package com.linkedin.datahub.graphql.resolvers.auth;
import com.google.common.collect.ImmutableList;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
import com.linkedin.datahub.graphql.exception.AuthorizationException;
@ -55,7 +56,7 @@ public class ListAccessTokensResolver implements DataFetcher<CompletableFuture<L
new SortCriterion().setField(EXPIRES_AT_FIELD_NAME).setOrder(SortOrder.DESCENDING);
final SearchResult searchResult = _entityClient.search(Constants.ACCESS_TOKEN_ENTITY_NAME, "",
buildFilter(filters), sortCriterion, start, count,
buildFilter(filters, Collections.emptyList()), sortCriterion, start, count,
getAuthentication(environment));
final List<AccessTokenMetadata> tokens = searchResult.getEntities().stream().map(entity -> {
@ -94,6 +95,6 @@ public class ListAccessTokensResolver implements DataFetcher<CompletableFuture<L
*/
private boolean isListingSelfTokens(final List<FacetFilterInput> filters, final QueryContext context) {
return AuthorizationUtils.canGeneratePersonalAccessToken(context) && filters.stream()
.anyMatch(filter -> filter.getField().equals("ownerUrn") && filter.getValue().equals(context.getActorUrn()));
.anyMatch(filter -> filter.getField().equals("ownerUrn") && filter.getValues().equals(ImmutableList.of(context.getActorUrn())));
}
}

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java
View File

@ -12,7 +12,6 @@ import com.linkedin.metadata.aspect.EnvelopedAspect;
import com.linkedin.metadata.authorization.PoliciesConfig;
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
import com.linkedin.metadata.query.filter.Criterion;
import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.r2.RemoteInvocationException;
@ -111,7 +110,7 @@ public class TimeSeriesAspectResolver implements DataFetcher<CompletableFuture<L
return null;
}
return new Filter().setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(maybeFilters.getAnd().stream()
.map(filter -> new Criterion().setField(filter.getField()).setValue(filter.getValue()))
.map(filter -> criterionFromFilter(filter))
.collect(Collectors.toList())))));
}
}

9
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java
View File

@ -1,5 +1,6 @@
package com.linkedin.datahub.graphql.resolvers.recommendation;
import com.google.common.collect.ImmutableList;
import com.linkedin.common.urn.Urn;
import com.linkedin.datahub.graphql.generated.ContentParams;
import com.linkedin.datahub.graphql.generated.EntityProfileParams;
@ -14,7 +15,6 @@ import com.linkedin.datahub.graphql.generated.RecommendationRequestContext;
import com.linkedin.datahub.graphql.generated.SearchParams;
import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper;
import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
import com.linkedin.metadata.query.filter.Criterion;
import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.recommendation.EntityRequestContext;
import com.linkedin.metadata.recommendation.RecommendationsService;
@ -31,7 +31,7 @@ import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*;
@Slf4j
@ -88,7 +88,7 @@ public class ListRecommendationsResolver implements DataFetcher<CompletableFutur
searchRequestContext.setFilters(new CriterionArray(requestContext.getSearchRequestContext()
.getFilters()
.stream()
.map(facetField -> new Criterion().setField(facetField.getField()).setValue(facetField.getValue()))
.map(facetField -> criterionFromFilter(facetField))
.collect(Collectors.toList())));
}
mappedRequestContext.setSearchRequestContext(searchRequestContext);
@ -148,7 +148,8 @@ public class ListRecommendationsResolver implements DataFetcher<CompletableFutur
searchParams.setFilters(params.getSearchParams()
.getFilters()
.stream()
.map(criterion -> Filter.builder().setField(criterion.getField()).setValue(criterion.getValue()).build())
.map(criterion -> Filter.builder().setField(criterion.getField()).setValues(
ImmutableList.of(criterion.getValue())).build())
.collect(Collectors.toList()));
}
mappedParams.setSearchParams(searchParams);

2
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java
View File

@ -52,7 +52,7 @@ public class SearchAcrossEntitiesResolver implements DataFetcher<CompletableFutu
"Executing search for multiple entities: entity types {}, query {}, filters: {}, start: {}, count: {}",
input.getTypes(), input.getQuery(), input.getFilters(), start, count);
return UrnSearchResultsMapper.map(_entityClient.searchAcrossEntities(entityNames, sanitizedQuery,
ResolverUtils.buildFilter(input.getFilters()), start, count, ResolverUtils.getAuthentication(environment)));
ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()), start, count, ResolverUtils.getAuthentication(environment)));
} catch (Exception e) {
log.error(
"Failed to execute search for multiple entities: entity types {}, query {}, filters: {}, start: {}, count: {}",

4
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java
View File

@ -72,7 +72,7 @@ public class SearchAcrossLineageResolver
urn, resolvedDirection, input.getTypes(), input.getQuery(), filters, start, count);
return UrnSearchAcrossLineageResultsMapper.map(
_entityClient.searchAcrossLineage(urn, resolvedDirection, entityNames, sanitizedQuery,
maxHops, ResolverUtils.buildFilter(filters), null, start, count,
maxHops, ResolverUtils.buildFilter(filters, input.getOrFilters()), null, start, count,
ResolverUtils.getAuthentication(environment)));
} catch (RemoteInvocationException e) {
log.error(
@ -89,7 +89,7 @@ public class SearchAcrossLineageResolver
private Integer getMaxHops(List<FacetFilterInput> filters) {
Set<String> degreeFilterValues = filters.stream()
.filter(filter -> filter.getField().equals("degree"))
.map(FacetFilterInput::getValue)
.flatMap(filter -> filter.getValues().stream())
.collect(Collectors.toSet());
Integer maxHops = null;
if (!degreeFilterValues.contains("3+")) {

14
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java
View File

@ -41,17 +41,17 @@ public class SearchResolver implements DataFetcher<CompletableFuture<SearchResul
return CompletableFuture.supplyAsync(() -> {
try {
log.debug("Executing search. entity type {}, query {}, filters: {}, start: {}, count: {}", input.getType(),
input.getQuery(), input.getFilters(), start, count);
log.debug("Executing search. entity type {}, query {}, filters: {}, orFilters: {}, start: {}, count: {}", input.getType(),
input.getQuery(), input.getFilters(), input.getOrFilters(), start, count);
return UrnSearchResultsMapper.map(
_entityClient.search(entityName, sanitizedQuery, ResolverUtils.buildFilter(input.getFilters()), null, start,
_entityClient.search(entityName, sanitizedQuery, ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()), null, start,
count, ResolverUtils.getAuthentication(environment)));
} catch (Exception e) {
log.error("Failed to execute search: entity type {}, query {}, filters: {}, start: {}, count: {}",
input.getType(), input.getQuery(), input.getFilters(), start, count);
log.error("Failed to execute search: entity type {}, query {}, filters: {}, orFilters: {}, start: {}, count: {}",
input.getType(), input.getQuery(), input.getFilters(), input.getOrFilters(), start, count);
throw new RuntimeException(
"Failed to execute search: " + String.format("entity type %s, query %s, filters: %s, start: %s, count: %s",
input.getType(), input.getQuery(), input.getFilters(), start, count), e);
"Failed to execute search: " + String.format("entity type %s, query %s, filters: %s, orFilters: %s, start: %s, count: %s",
input.getType(), input.getQuery(), input.getFilters(), input.getOrFilters(), start, count), e);
}
});
}

24
datahub-graphql-core/src/main/resources/recommendation.graphql
View File

@ -217,7 +217,7 @@ type SearchParams {
"""
Entity types to be searched. If this is not provided, all entities will be searched.
"""
types: [EntityType!]
types: [EntityType!]
"""
Search query
@ -237,12 +237,22 @@ type Filter {
"""
Name of field to filter by
"""
field: String!
field: String!
"""
Value of the field to filter by
"""
value: String!
"""
Values, one of which the intended field should match.
"""
values: [String!]!
"""
If the filter should or should not be matched
"""
negated: Boolean
"""
Condition for the values. How to If unset, assumed to be equality
"""
condition: FilterOperator
}
"""
@ -269,4 +279,4 @@ type ContentParams {
Number of entities corresponding to the recommended content
"""
count: Long
}
}

65
datahub-graphql-core/src/main/resources/search.graphql
View File

@ -65,9 +65,15 @@ input SearchInput {
count: Int
"""
Facet filters to apply to search results
Deprecated in favor of the more expressive orFilters field
Facet filters to apply to search results. These will be 'AND'-ed together.
"""
filters: [FacetFilterInput!]
filters: [FacetFilterInput!] @deprecated(reason: "Use `orFilters`- they are more expressive")
"""
A list of disjunctive criterion for the filter. (or operation to combine filters)
"""
orFilters: [OrFilter!]
}
"""
@ -95,9 +101,15 @@ input SearchAcrossEntitiesInput {
count: Int
"""
Faceted filters applied to search results
Deprecated in favor of the more expressive orFilters field
Facet filters to apply to search results. These will be 'AND'-ed together.
"""
filters: [FacetFilterInput!]
filters: [FacetFilterInput!] @deprecated(reason: "Use `orFilters`- they are more expressive")
"""
A list of disjunctive criterion for the filter. (or operation to combine filters)
"""
orFilters: [OrFilter!]
}
"""
@ -135,9 +147,25 @@ input SearchAcrossLineageInput {
count: Int
"""
Faceted filters applied to search results
Deprecated in favor of the more expressive orFilters field
Facet filters to apply to search results. These will be 'AND'-ed together.
"""
filters: [FacetFilterInput!]
filters: [FacetFilterInput!] @deprecated(reason: "Use `orFilters`- they are more expressive")
"""
A list of disjunctive criterion for the filter. (or operation to combine filters)
"""
orFilters: [OrFilter!]
}
"""
A list of disjunctive criterion for the filter. (or operation to combine filters)
"""
input OrFilter {
"""
A list of and criteria the filter applies to the query
"""
and: [FacetFilterInput!]
}
"""
@ -150,14 +178,9 @@ input FacetFilterInput {
field: String!
"""
Value of the field to filter by (soon to be deprecated)
Values, one of which the intended field should match.
"""
value: String!
"""
Values of the field to filter by
"""
values: [String!]
values: [String!]!
"""
If the filter should or should not be matched
@ -165,12 +188,12 @@ input FacetFilterInput {
negated: Boolean
"""
Condition for the values. If unset, assumed to be equality
Condition for the values. How to If unset, assumed to be equality
"""
condition: SearchCondition
condition: FilterOperator
}
enum SearchCondition {
enum FilterOperator {
"""
Represent the relation: String field contains value, e.g. name contains Profile
"""
@ -508,9 +531,15 @@ input BrowseInput {
count: Int
"""
Faceted filters applied to browse results
Deprecated in favor of the more expressive orFilters field
Facet filters to apply to search results. These will be 'AND'-ed together.
"""
filters: [FacetFilterInput!]
filters: [FacetFilterInput!] @deprecated(reason: "Use `orFilters`- they are more expressive")
"""
A list of disjunctive criterion for the filter. (or operation to combine filters)
"""
orFilters: [OrFilter!]
}
"""

11
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java
View File

@ -6,10 +6,10 @@ import com.linkedin.datahub.graphql.TestUtils;
import com.linkedin.datahub.graphql.generated.FacetFilterInput;
import com.linkedin.datahub.graphql.generated.ListAccessTokenInput;
import com.linkedin.datahub.graphql.generated.ListAccessTokenResult;
import com.linkedin.datahub.graphql.generated.SearchCondition;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.metadata.Constants;
import graphql.schema.DataFetchingEnvironment;
import java.util.Collections;
import junit.framework.TestCase;
import org.mockito.Mockito;
@ -27,15 +27,18 @@ public class ListAccessTokensResolverTest extends TestCase {
final ListAccessTokenInput input = new ListAccessTokenInput();
input.setStart(0);
input.setCount(100);
final ImmutableList<FacetFilterInput> filters = ImmutableList.of(new FacetFilterInput("actor",
"urn:li:corpuser:test", ImmutableList.of("urn:li:corpuser:test"), false, SearchCondition.EQUAL));
FacetFilterInput filter = new FacetFilterInput();
filter.setField("actor");
filter.setValues(ImmutableList.of("urn:li:corpuser:test"));
final ImmutableList<FacetFilterInput> filters = ImmutableList.of(filter);
input.setFilters(filters);
Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input);
final EntityClient mockClient = Mockito.mock(EntityClient.class);
Mockito.when(mockClient.filter(
Mockito.eq(Constants.ACCESS_TOKEN_ENTITY_NAME),
Mockito.eq(buildFilter(filters)),
Mockito.eq(buildFilter(filters, Collections.emptyList())),
Mockito.notNull(),
Mockito.eq(input.getStart()),
Mockito.eq(input.getCount()),

BIN
datahub-web-react/public/meta-favicon.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 237 KiB

112
datahub-web-react/src/Mocks.tsx
View File

@ -27,6 +27,7 @@ import {
RelationshipDirection,
Container,
PlatformPrivileges,
FilterOperator,
} from './types.generated';
import { GetTagDocument } from './graphql/tag.generated';
import { GetMlModelDocument } from './graphql/mlModel.generated';
@ -1701,7 +1702,8 @@ export const mocks = [
path: [],
start: 0,
count: 20,
filters: null,
filters: [],
orFilters: [],
},
},
},
@ -1735,7 +1737,8 @@ export const mocks = [
path: ['prod', 'hdfs'],
start: 0,
count: 20,
filters: null,
filters: [],
orFilters: [],
},
},
},
@ -1769,7 +1772,8 @@ export const mocks = [
path: ['prod'],
start: 0,
count: 20,
filters: null,
filters: [],
orFilters: [],
},
},
},
@ -1873,6 +1877,7 @@ export const mocks = [
start: 0,
count: 10,
filters: [],
orFilters: [],
},
},
},
@ -1946,10 +1951,17 @@ export const mocks = [
query: 'test',
start: 0,
count: 10,
filters: [
filters: [],
orFilters: [
{
field: 'platform',
value: 'kafka',
and: [
{
field: 'platform',
values: ['kafka'],
negated: false,
condition: FilterOperator.Equal,
},
],
},
],
},
@ -2019,6 +2031,7 @@ export const mocks = [
start: 0,
count: 1,
filters: [],
orFilters: [],
},
},
},
@ -2109,14 +2122,17 @@ export const mocks = [
query: 'test',
start: 0,
count: 10,
filters: [
filters: [],
orFilters: [
{
field: 'platform',
value: 'kafka',
},
{
field: 'platform',
value: 'hdfs',
and: [
{
field: 'platform',
values: ['kafka', 'hdfs'],
negated: false,
condition: FilterOperator.Equal,
},
],
},
],
},
@ -2256,6 +2272,7 @@ export const mocks = [
start: 0,
count: 1,
filters: [],
orFilters: [],
},
},
},
@ -2283,6 +2300,7 @@ export const mocks = [
start: 0,
count: 1,
filters: [],
orFilters: [],
},
},
},
@ -2348,6 +2366,7 @@ export const mocks = [
start: 0,
count: 20,
filters: [],
orFilters: [],
},
},
},
@ -2421,6 +2440,7 @@ export const mocks = [
start: 0,
count: 10,
filters: [],
orFilters: [],
},
},
},
@ -2564,6 +2584,7 @@ export const mocks = [
start: 0,
count: 10,
filters: [],
orFilters: [],
},
},
},
@ -2637,10 +2658,17 @@ export const mocks = [
query: 'test',
start: 0,
count: 10,
filters: [
filters: [],
orFilters: [
{
field: 'platform',
value: 'kafka',
and: [
{
field: 'platform',
values: ['kafka'],
negated: false,
condition: FilterOperator.Equal,
},
],
},
],
},
@ -2738,10 +2766,17 @@ export const mocks = [
query: 'test',
start: 0,
count: 10,
filters: [
filters: [],
orFilters: [
{
field: 'platform',
value: 'kafka',
and: [
{
field: 'platform',
values: ['kafka'],
negated: false,
condition: FilterOperator.Equal,
},
],
},
],
},
@ -2780,6 +2815,7 @@ export const mocks = [
start: 0,
count: 10,
filters: [],
orFilters: [],
},
},
},
@ -2841,6 +2877,7 @@ export const mocks = [
start: 0,
count: 1,
filters: [],
orFilters: [],
},
},
},
@ -2906,6 +2943,7 @@ export const mocks = [
start: 0,
count: 20,
filters: [],
orFilters: [],
},
},
},
@ -2978,14 +3016,17 @@ export const mocks = [
query: 'test',
start: 0,
count: 10,
filters: [
filters: [],
orFilters: [
{
field: 'platform',
value: 'kafka',
},
{
field: 'platform',
value: 'hdfs',
and: [
{
field: 'platform',
values: ['kafka', 'hdfs'],
negated: false,
condition: FilterOperator.Equal,
},
],
},
],
},
@ -3052,14 +3093,17 @@ export const mocks = [
query: 'test',
start: 0,
count: 10,
filters: [
filters: [],
orFilters: [
{
field: 'platform',
value: 'kafka',
},
{
field: 'platform',
value: 'hdfs',
and: [
{
field: 'platform',
values: ['kafka', 'hdfs'],
negated: false,
condition: FilterOperator.Equal,
},
],
},
],
},
@ -3251,6 +3295,7 @@ export const mocks = [
start: 0,
count: 10,
filters: [],
orFilters: [],
},
},
},
@ -3307,6 +3352,7 @@ export const mocks = [
start: 0,
count: 6,
filters: [],
orFilters: [],
},
},
},

2
datahub-web-react/src/app/entity/container/ContainerEntitiesTab.tsx
View File

@ -7,7 +7,7 @@ export const ContainerEntitiesTab = () => {
const fixedFilter = {
field: 'container',
value: urn,
values: [urn],
};
return (

2
datahub-web-react/src/app/entity/group/GroupAssets.tsx
View File

@ -14,7 +14,7 @@ export const GroupAssets = ({ urn }: Props) => {
return (
<GroupAssetsWrapper>
<EmbeddedListSearchSection
fixedFilter={{ field: 'owners', value: urn }}
fixedFilter={{ field: 'owners', values: [urn] }}
emptySearchQuery="*"
placeholderText="Filter entities..."
/>

17
datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx
View File

@ -2,7 +2,7 @@ import React, { useState, useEffect } from 'react';
import styled from 'styled-components';
import { ApolloError } from '@apollo/client';
import { EntityType, FacetFilterInput } from '../../../../../../types.generated';
import { ENTITY_FILTER_NAME } from '../../../../../search/utils/constants';
import { ENTITY_FILTER_NAME, UnionType } from '../../../../../search/utils/constants';
import { SearchCfg } from '../../../../../../conf';
import { EmbeddedListSearchResults } from './EmbeddedListSearchResults';
import EmbeddedListSearchHeader from './EmbeddedListSearchHeader';
@ -11,6 +11,7 @@ import { GetSearchResultsParams, SearchResultsInterface } from './types';
import { isListSubset } from '../../../utils';
import { EntityAndType } from '../../../types';
import { Message } from '../../../../../shared/Message';
import { generateOrFilters } from '../../../../../search/utils/generateOrFilters';
const Container = styled.div`
display: flex;
@ -48,10 +49,12 @@ export const addFixedQuery = (baseQuery: string, fixedQuery: string, emptyQuery:
type Props = {
query: string;
page: number;
unionType: UnionType;
filters: FacetFilterInput[];
onChangeQuery: (query) => void;
onChangeFilters: (filters) => void;
onChangePage: (page) => void;
onChangeUnionType: (unionType: UnionType) => void;
emptySearchQuery?: string | null;
fixedFilter?: FacetFilterInput | null;
fixedQuery?: string | null;
@ -72,9 +75,11 @@ export const EmbeddedListSearch = ({
query,
filters,
page,
unionType,
onChangeQuery,
onChangeFilters,
onChangePage,
onChangeUnionType,
emptySearchQuery,
fixedFilter,
fixedQuery,
@ -95,7 +100,7 @@ export const EmbeddedListSearch = ({
const finalFilters = (fixedFilter && [...filtersWithoutEntities, fixedFilter]) || filtersWithoutEntities;
const entityFilters: Array<EntityType> = filters
.filter((filter) => filter.field === ENTITY_FILTER_NAME)
.map((filter) => filter.value.toUpperCase() as EntityType);
.flatMap((filter) => filter.values.map((value) => value?.toUpperCase() as EntityType));
const [showFilters, setShowFilters] = useState(defaultShowFilters || false);
const [isSelectMode, setIsSelectMode] = useState(false);
@ -109,7 +114,8 @@ export const EmbeddedListSearch = ({
query: finalQuery,
start: (page - 1) * SearchCfg.RESULTS_PER_PAGE,
count: SearchCfg.RESULTS_PER_PAGE,
filters: finalFilters,
filters: [],
orFilters: generateOrFilters(unionType, filtersWithoutEntities),
},
},
skip: true,
@ -126,7 +132,8 @@ export const EmbeddedListSearch = ({
query: finalQuery,
start: (page - 1) * numResultsPerPage,
count: numResultsPerPage,
filters: finalFilters,
filters: [],
orFilters: generateOrFilters(unionType, filtersWithoutEntities),
},
},
});
@ -200,12 +207,14 @@ export const EmbeddedListSearch = ({
searchBarInputStyle={searchBarInputStyle}
/>
<EmbeddedListSearchResults
unionType={unionType}
loading={loading}
searchResponse={data}
filters={filteredFilters}
selectedFilters={filters}
onChangeFilters={onChangeFilters}
onChangePage={onChangePage}
onChangeUnionType={onChangeUnionType}
page={page}
showFilters={showFilters}
numResultsPerPage={numResultsPerPage}

5
datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchModal.tsx
View File

@ -3,6 +3,7 @@ import { Button, Modal } from 'antd';
import styled from 'styled-components';
import { FacetFilterInput } from '../../../../../../types.generated';
import { EmbeddedListSearch } from './EmbeddedListSearch';
import { UnionType } from '../../../../../search/utils/constants';
const SearchContainer = styled.div`
height: 500px;
@ -41,6 +42,8 @@ export const EmbeddedListSearchModal = ({
// Component state
const [query, setQuery] = useState<string>('');
const [page, setPage] = useState(1);
const [unionType, setUnionType] = useState(UnionType.AND);
const [filters, setFilters] = useState<Array<FacetFilterInput>>([]);
const onChangeQuery = (q: string) => {
@ -70,9 +73,11 @@ export const EmbeddedListSearchModal = ({
query={query}
filters={filters}
page={page}
unionType={unionType}
onChangeQuery={onChangeQuery}
onChangeFilters={onChangeFilters}
onChangePage={onChangePage}
onChangeUnionType={setUnionType}
emptySearchQuery={emptySearchQuery}
fixedFilter={fixedFilter}
fixedQuery={fixedQuery}

46
datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx
View File

@ -2,11 +2,12 @@ import React from 'react';
import { Pagination, Typography } from 'antd';
import styled from 'styled-components';
import { FacetFilterInput, FacetMetadata, SearchResults as SearchResultType } from '../../../../../../types.generated';
import { SearchFilters } from '../../../../../search/SearchFilters';
import { SearchCfg } from '../../../../../../conf';
import { EntityNameList } from '../../../../../recommendations/renderer/component/EntityNameList';
import { ReactComponent as LoadingSvg } from '../../../../../../images/datahub-logo-color-loading_pendulum.svg';
import { EntityAndType } from '../../../types';
import { UnionType } from '../../../../../search/utils/constants';
import { SearchFiltersSection } from '../../../../../search/SearchFiltersSection';
const SearchBody = styled.div`
height: 100%;
@ -44,33 +45,11 @@ const PaginationInfoContainer = styled.span`
align-items: center;
`;
const FiltersHeader = styled.div`
font-size: 14px;
font-weight: 600;
flex: 0 0 auto;
padding-left: 20px;
padding-right: 20px;
padding-bottom: 8px;
width: 100%;
height: 46px;
line-height: 46px;
border-bottom: 1px solid;
border-color: ${(props) => props.theme.styles['border-color-base']};
`;
const StyledPagination = styled(Pagination)`
margin: 0px;
padding: 0px;
`;
const SearchFilterContainer = styled.div`
padding-top: 10px;
flex: 1 1 auto;
overflow: hidden;
`;
const LoadingContainer = styled.div`
padding-top: 40px;
padding-bottom: 40px;
@ -86,8 +65,10 @@ interface Props {
selectedFilters: Array<FacetFilterInput>;
loading: boolean;
showFilters?: boolean;
unionType: UnionType;
onChangeFilters: (filters: Array<FacetFilterInput>) => void;
onChangePage: (page: number) => void;
onChangeUnionType: (unionType: UnionType) => void;
isSelectMode: boolean;
selectedEntities: EntityAndType[];
setSelectedEntities: (entities: EntityAndType[]) => any;
@ -102,6 +83,8 @@ export const EmbeddedListSearchResults = ({
selectedFilters,
loading,
showFilters,
unionType,
onChangeUnionType,
onChangeFilters,
onChangePage,
isSelectMode,
@ -120,15 +103,14 @@ export const EmbeddedListSearchResults = ({
<SearchBody>
{!!showFilters && (
<FiltersContainer>
<FiltersHeader>Filter</FiltersHeader>
<SearchFilterContainer>
<SearchFilters
loading={loading}
facets={filters || []}
selectedFilters={selectedFilters}
onFilterSelect={(newFilters) => onChangeFilters(newFilters)}
/>
</SearchFilterContainer>
<SearchFiltersSection
filters={filters}
selectedFilters={selectedFilters}
unionType={unionType}
loading={loading}
onChangeFilters={onChangeFilters}
onChangeUnionType={onChangeUnionType}
/>
</FiltersContainer>
)}
<ResultContainer>

20
datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx
View File

@ -8,6 +8,7 @@ import { navigateToEntitySearchUrl } from './navigateToEntitySearchUrl';
import { GetSearchResultsParams, SearchResultsInterface } from './types';
import { useEntityQueryParams } from '../../../containers/profile/utils';
import { EmbeddedListSearch } from './EmbeddedListSearch';
import { UnionType } from '../../../../../search/utils/constants';
type Props = {
emptySearchQuery?: string | null;
@ -44,6 +45,8 @@ export const EmbeddedListSearchSection = ({
const params = QueryString.parse(location.search, { arrayFormat: 'comma' });
const query: string = params?.query as string;
const page: number = params.page && Number(params.page as string) > 0 ? Number(params.page as string) : 1;
const unionType: UnionType = Number(params.unionType as any as UnionType) || UnionType.AND;
const filters: Array<FacetFilterInput> = useFilters(params);
const onSearch = (q: string) => {
@ -54,6 +57,7 @@ export const EmbeddedListSearchSection = ({
page: 1,
filters,
history,
unionType,
});
};
@ -65,6 +69,7 @@ export const EmbeddedListSearchSection = ({
page: 1,
filters: newFilters,
history,
unionType,
});
};
@ -76,6 +81,19 @@ export const EmbeddedListSearchSection = ({
page: newPage,
filters,
history,
unionType,
});
};
const onChangeUnionType = (newUnionType: UnionType) => {
navigateToEntitySearchUrl({
baseUrl: location.pathname,
baseParams,
query,
page,
filters,
history,
unionType: newUnionType,
});
};
@ -83,10 +101,12 @@ export const EmbeddedListSearchSection = ({
<EmbeddedListSearch
query={query || ''}
page={page}
unionType={unionType}
filters={filters}
onChangeFilters={onChangeFilters}
onChangeQuery={onSearch}
onChangePage={onChangePage}
onChangeUnionType={onChangeUnionType}
emptySearchQuery={emptySearchQuery}
fixedFilter={fixedFilter}
fixedQuery={fixedQuery}

7
datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelect.tsx
View File

@ -5,7 +5,7 @@ import { FilterOutlined } from '@ant-design/icons';
import { useEntityRegistry } from '../../../../../useEntityRegistry';
import { EntityType, FacetFilterInput } from '../../../../../../types.generated';
import { ENTITY_FILTER_NAME } from '../../../../../search/utils/constants';
import { ENTITY_FILTER_NAME, UnionType } from '../../../../../search/utils/constants';
import { SearchCfg } from '../../../../../../conf';
import { EmbeddedListSearchResults } from './EmbeddedListSearchResults';
import { useGetSearchResultsForMultipleQuery } from '../../../../../../graphql/search.generated';
@ -61,6 +61,7 @@ export const SearchSelect = ({ fixedEntityTypes, placeholderText, selectedEntiti
const [query, setQuery] = useState<string>('');
const [page, setPage] = useState(1);
const [filters, setFilters] = useState<Array<FacetFilterInput>>([]);
const [unionType, setUnionType] = useState(UnionType.AND);
const [showFilters, setShowFilters] = useState(false);
const [numResultsPerPage, setNumResultsPerPage] = useState(SearchCfg.RESULTS_PER_PAGE);
@ -70,7 +71,7 @@ export const SearchSelect = ({ fixedEntityTypes, placeholderText, selectedEntiti
);
const entityFilters: Array<EntityType> = filters
.filter((filter) => filter.field === ENTITY_FILTER_NAME)
.map((filter) => filter.value.toUpperCase() as EntityType);
.flatMap((filter) => filter.values.map((value) => value.toUpperCase() as EntityType));
const finalEntityTypes = (entityFilters.length > 0 && entityFilters) || fixedEntityTypes || [];
// Execute search
@ -166,9 +167,11 @@ export const SearchSelect = ({ fixedEntityTypes, placeholderText, selectedEntiti
loading={loading}
searchResponse={searchAcrossEntities}
filters={facets}
unionType={unionType}
selectedFilters={filters}
onChangeFilters={onChangeFilters}
onChangePage={onChangePage}
onChangeUnionType={setUnionType}
page={page}
showFilters={showFilters}
numResultsPerPage={numResultsPerPage}

6
datahub-web-react/src/app/entity/shared/components/styled/search/navigateToEntitySearchUrl.ts
View File

@ -2,6 +2,7 @@ import { RouteComponentProps } from 'react-router';
import * as QueryString from 'query-string';
import { EntityType, FacetFilterInput } from '../../../../../../types.generated';
import filtersToQueryStringParams from '../../../../../search/utils/filtersToQueryStringParams';
import { UnionType } from '../../../../../search/utils/constants';
export const navigateToEntitySearchUrl = ({
baseUrl,
@ -11,6 +12,7 @@ export const navigateToEntitySearchUrl = ({
page: newPage = 1,
filters: newFilters,
history,
unionType,
}: {
baseUrl: string;
baseParams: Record<string, string | boolean>;
@ -19,10 +21,11 @@ export const navigateToEntitySearchUrl = ({
page?: number;
filters?: Array<FacetFilterInput>;
history: RouteComponentProps['history'];
unionType: UnionType;
}) => {
const constructedFilters = newFilters || [];
if (newType) {
constructedFilters.push({ field: 'entity', value: newType });
constructedFilters.push({ field: 'entity', values: [newType] });
}
const search = QueryString.stringify(
@ -30,6 +33,7 @@ export const navigateToEntitySearchUrl = ({
...filtersToQueryStringParams(constructedFilters),
query: newQuery,
page: newPage,
unionType,
...baseParams,
},
{ arrayFormat: 'comma' },

4
datahub-web-react/src/app/entity/shared/tabs/Lineage/ImpactAnalysis.tsx
View File

@ -33,7 +33,7 @@ export const ImpactAnalysis = ({ urn, direction }: Props) => {
);
const entityFilters: Array<EntityType> = filters
.filter((filter) => filter.field === ENTITY_FILTER_NAME)
.map((filter) => filter.value.toUpperCase() as EntityType);
.flatMap((filter) => filter.values.map((value) => value.toUpperCase() as EntityType));
const { data, loading } = useSearchAcrossLineageQuery({
variables: {
@ -67,7 +67,7 @@ export const ImpactAnalysis = ({ urn, direction }: Props) => {
direction,
})}
defaultShowFilters
defaultFilters={[{ field: 'degree', value: '1' }]}
defaultFilters={[{ field: 'degree', values: ['1'] }]}
/>
</ImpactAnalysisWrapper>
);

2
datahub-web-react/src/app/entity/user/UserAssets.tsx
View File

@ -15,7 +15,7 @@ export const UserAssets = ({ urn }: Props) => {
return (
<UserAssetsWrapper>
<EmbeddedListSearchSection
fixedFilter={{ field: 'owners', value: urn }}
fixedFilter={{ field: 'owners', values: [urn] }}
emptySearchQuery="*"
placeholderText="Filter entities..."
/>

1
datahub-web-react/src/app/home/HomePageHeader.tsx
View File

@ -166,6 +166,7 @@ export const HomePageHeader = () => {
start: 0,
count: 6,
filters: [],
orFilters: [],
},
},
});

4
datahub-web-react/src/app/ingest/source/IngestedAssets.tsx
View File

@ -71,7 +71,7 @@ export default function IngestedAssets({ id }: Props) {
filters: [
{
field: 'runId',
value: id,
values: [id],
},
],
},
@ -135,7 +135,7 @@ export default function IngestedAssets({ id }: Props) {
{showAssetSearch && (
<EmbeddedListSearchModal
searchBarStyle={{ width: 600, marginRight: 40 }}
fixedFilter={{ field: 'runId', value: id }}
fixedFilter={{ field: 'runId', values: [id] }}
onClose={() => setShowAssetSearch(false)}
/>
)}

2
datahub-web-react/src/app/recommendations/renderer/component/GlossaryTermSearchList.tsx
View File

@ -53,7 +53,7 @@ export const GlossaryTermSearchList = ({ content, onClick }: Props) => {
filters: [
{
field: 'glossaryTerms',
value: term.urn,
values: [term.urn],
},
],
history,

2
datahub-web-react/src/app/recommendations/renderer/component/TagSearchList.tsx
View File

@ -44,7 +44,7 @@ export const TagSearchList = ({ content, onClick }: Props) => {
filters: [
{
field: 'tags',
value: tag.urn,
values: [tag.urn],
},
],
history,

2
datahub-web-react/src/app/search/AdvancedSearchAddFilterSelect.tsx
View File

@ -37,11 +37,13 @@ export const AdvancedSearchAddFilterSelect = ({ selectedFilters, onFilterFieldSe
>
{Object.keys(FIELD_TO_LABEL)
.sort((a, b) => FIELD_TO_LABEL[a].localeCompare(FIELD_TO_LABEL[b]))
.filter((key) => key !== 'degree')
.map((key) => (
<Option
// disable the `entity` option if they already have an entity filter selected
disabled={key === 'entity' && !!selectedFilters.find((filter) => filter.field === 'entity')}
value={key}
data-testid={`adv-search-add-filter-${key}`}
>
{FIELD_TO_LABEL[key]}
</Option>

6
datahub-web-react/src/app/search/AdvancedSearchFilter.tsx
View File

@ -14,6 +14,7 @@ type Props = {
filter: FacetFilterInput;
onClose: () => void;
onUpdate: (newValue: FacetFilterInput) => void;
loading: boolean;
};
const FilterContainer = styled.div`
@ -46,7 +47,7 @@ const FilterFieldLabel = styled.span`
margin-right: 2px;
`;
export const AdvancedSearchFilter = ({ facet, filter, onClose, onUpdate }: Props) => {
export const AdvancedSearchFilter = ({ facet, filter, onClose, onUpdate, loading }: Props) => {
const [isEditing, setIsEditing] = useState(false);
return (
<>
@ -73,7 +74,7 @@ export const AdvancedSearchFilter = ({ facet, filter, onClose, onUpdate }: Props
<CloseOutlined />
</CloseSpan>
</FieldFilterSection>
<AdvancedSearchFilterValuesSection filter={filter} facet={facet} />
{!loading && <AdvancedSearchFilterValuesSection filter={filter} facet={facet} />}
</FilterContainer>
{isEditing && (
<AdvancedFilterSelectValueModal
@ -83,7 +84,6 @@ export const AdvancedSearchFilter = ({ facet, filter, onClose, onUpdate }: Props
onSelect={(values) => {
const newFilter: FacetFilterInput = {
field: filter.field,
value: '',
values: values as string[],
condition: filter.condition,
negated: filter.negated,

2
datahub-web-react/src/app/search/AdvancedSearchFilterValuesSection.tsx
View File

@ -30,7 +30,7 @@ export const AdvancedSearchFilterValuesSection = ({ facet, filter }: Props) => {
return (
<StyledSearchFilterLabel>
<SearchFilterLabel hideCount aggregation={matchedAggregation} field={value} />
<SearchFilterLabel hideCount aggregation={matchedAggregation} field={filter.field} />
</StyledSearchFilterLabel>
);
})}

10
datahub-web-react/src/app/search/AdvancedSearchFilters.tsx
View File

@ -2,7 +2,7 @@ import * as React from 'react';
import { useState } from 'react';
import styled from 'styled-components';
import { FacetFilterInput, FacetMetadata, SearchCondition } from '../../types.generated';
import { FacetFilterInput, FacetMetadata, FilterOperator } from '../../types.generated';
import { ANTD_GRAY } from '../entity/shared/constants';
import { AdvancedSearchFilter } from './AdvancedSearchFilter';
import { AdvancedSearchFilterOverallUnionTypeSelect } from './AdvancedSearchFilterOverallUnionTypeSelect';
@ -47,6 +47,7 @@ interface Props {
onFilterSelect: (newFilters: Array<FacetFilterInput>) => void;
onChangeUnionType: (unionType: UnionType) => void;
unionType?: UnionType;
loading: boolean;
}
export const AdvancedSearchFilters = ({
@ -55,6 +56,7 @@ export const AdvancedSearchFilters = ({
selectedFilters,
onFilterSelect,
onChangeUnionType,
loading,
}: Props) => {
const [filterField, setFilterField] = useState<null | string>(null);
@ -68,10 +70,9 @@ export const AdvancedSearchFilters = ({
const newFilter: FacetFilterInput = {
field: filterField,
values: values as string[],
value: '', // TODO(Gabe): remove once we refactor the model
condition: FIELDS_THAT_USE_CONTAINS_OPERATOR.includes(filterField)
? SearchCondition.Contain
: SearchCondition.Equal,
? FilterOperator.Contain
: FilterOperator.Equal,
};
onFilterSelect([...selectedFilters, newFilter]);
};
@ -94,6 +95,7 @@ export const AdvancedSearchFilters = ({
{selectedFilters.map((filter) => (
<AdvancedSearchFilter
facet={facets.find((facet) => facet.field === filter.field) || facets[0]}
loading={loading}
filter={filter}
onClose={() => {
onFilterSelect(selectedFilters.filter((f) => f !== filter));

8
datahub-web-react/src/app/search/EditTextModal.tsx
View File

@ -21,13 +21,17 @@ export const EditTextModal = ({ defaultValue, onCloseModal, onOk, title }: Props
<Button onClick={onCloseModal} type="text">
Cancel
</Button>
<Button disabled={stagedValue.trim().length === 0} onClick={() => onOk?.(stagedValue)}>
<Button
data-testid="edit-text-done-btn"
disabled={stagedValue.trim().length === 0}
onClick={() => onOk?.(stagedValue)}
>
Done
</Button>
</>
}
>
<Input onChange={(e) => setStagedValue(e.target.value)} value={stagedValue} />
<Input data-testid="edit-text-input" onChange={(e) => setStagedValue(e.target.value)} value={stagedValue} />
</Modal>
);
};

99
datahub-web-react/src/app/search/SearchFiltersSection.tsx Normal file
View File

@ -0,0 +1,99 @@
import { Button } from 'antd';
import React, { useState } from 'react';
import styled from 'styled-components/macro';
import { FacetFilterInput, FacetMetadata } from '../../types.generated';
import { UnionType } from './utils/constants';
import { hasAdvancedFilters } from './utils/hasAdvancedFilters';
import { AdvancedSearchFilters } from './AdvancedSearchFilters';
import { SimpleSearchFilters } from './SimpleSearchFilters';
type Props = {
filters?: Array<FacetMetadata> | null;
selectedFilters: Array<FacetFilterInput>;
unionType: UnionType;
loading: boolean;
onChangeFilters: (filters: Array<FacetFilterInput>) => void;
onChangeUnionType: (unionType: UnionType) => void;
};
const FiltersContainer = styled.div`
display: block;
max-width: 260px;
min-width: 260px;
overflow-wrap: break-word;
border-right: 1px solid;
border-color: ${(props) => props.theme.styles['border-color-base']};
max-height: 100%;
`;
const FiltersHeader = styled.div`
font-size: 14px;
font-weight: 600;
padding-left: 20px;
padding-right: 20px;
padding-bottom: 8px;
width: 100%;
height: 47px;
line-height: 47px;
border-bottom: 1px solid;
border-color: ${(props) => props.theme.styles['border-color-base']};
justify-content: space-between;
display: flex;
`;
const SearchFilterContainer = styled.div`
padding-top: 10px;
`;
// This component renders the entire filters section that allows toggling
// between the simplified search experience and advanced search
export const SearchFiltersSection = ({
filters,
selectedFilters,
unionType,
loading,
onChangeFilters,
onChangeUnionType,
}: Props) => {
const onlyShowAdvancedFilters = hasAdvancedFilters(selectedFilters, unionType);
const [seeAdvancedFilters, setSeeAdvancedFilters] = useState(onlyShowAdvancedFilters);
return (
<FiltersContainer>
<FiltersHeader>
<span>Filter</span>
<span>
<Button
disabled={onlyShowAdvancedFilters}
type="link"
onClick={() => setSeeAdvancedFilters(!seeAdvancedFilters)}
>
{seeAdvancedFilters ? 'Filter' : 'Advanced'}
</Button>
</span>
</FiltersHeader>
{seeAdvancedFilters ? (
<AdvancedSearchFilters
unionType={unionType}
selectedFilters={selectedFilters}
onFilterSelect={(newFilters) => onChangeFilters(newFilters)}
onChangeUnionType={onChangeUnionType}
facets={filters || []}
loading={loading}
/>
) : (
<SearchFilterContainer>
<SimpleSearchFilters
loading={loading}
facets={filters || []}
selectedFilters={selectedFilters}
onFilterSelect={(newFilters) => onChangeFilters(newFilters)}
/>
</SearchFilterContainer>
)}
</FiltersContainer>
);
};

23
datahub-web-react/src/app/search/SearchPage.tsx
View File

@ -9,10 +9,11 @@ import { SearchResults } from './SearchResults';
import analytics, { EventType } from '../analytics';
import { useGetSearchResultsForMultipleQuery } from '../../graphql/search.generated';
import { SearchCfg } from '../../conf';
import { ENTITY_FILTER_NAME } from './utils/constants';
import { ENTITY_FILTER_NAME, UnionType } from './utils/constants';
import { GetSearchResultsParams } from '../entity/shared/components/styled/search/types';
import { EntityAndType } from '../entity/shared/types';
import { scrollToTop } from '../shared/searchUtils';
import { generateOrFilters } from './utils/generateOrFilters';
type SearchPageParams = {
type?: string;
@ -30,13 +31,15 @@ export const SearchPage = () => {
const query: string = decodeURIComponent(params.query ? (params.query as string) : '');
const activeType = entityRegistry.getTypeOrDefaultFromPathName(useParams<SearchPageParams>().type || '', undefined);
const page: number = params.page && Number(params.page as string) > 0 ? Number(params.page as string) : 1;
const unionType: UnionType = Number(params.unionType as any as UnionType) || UnionType.AND;
const filters: Array<FacetFilterInput> = useFilters(params);
const filtersWithoutEntities: Array<FacetFilterInput> = filters.filter(
(filter) => filter.field !== ENTITY_FILTER_NAME,
);
const entityFilters: Array<EntityType> = filters
.filter((filter) => filter.field === ENTITY_FILTER_NAME)
.map((filter) => filter.value.toUpperCase() as EntityType);
.flatMap((filter) => filter.values.map((value) => value?.toUpperCase() as EntityType));
const [numResultsPerPage, setNumResultsPerPage] = useState(SearchCfg.RESULTS_PER_PAGE);
const [isSelectMode, setIsSelectMode] = useState(false);
@ -54,7 +57,8 @@ export const SearchPage = () => {
query,
start: (page - 1) * numResultsPerPage,
count: numResultsPerPage,
filters: filtersWithoutEntities,
filters: [],
orFilters: generateOrFilters(unionType, filtersWithoutEntities),
},
},
});
@ -75,7 +79,8 @@ export const SearchPage = () => {
query,
start: (page - 1) * SearchCfg.RESULTS_PER_PAGE,
count: SearchCfg.RESULTS_PER_PAGE,
filters: filtersWithoutEntities,
filters: [],
orFilters: generateOrFilters(unionType, filtersWithoutEntities),
},
},
});
@ -85,12 +90,16 @@ export const SearchPage = () => {
};
const onChangeFilters = (newFilters: Array<FacetFilterInput>) => {
navigateToSearchUrl({ type: activeType, query, page: 1, filters: newFilters, history });
navigateToSearchUrl({ type: activeType, query, page: 1, filters: newFilters, history, unionType });
};
const onChangeUnionType = (newUnionType: UnionType) => {
navigateToSearchUrl({ type: activeType, query, page: 1, filters, history, unionType: newUnionType });
};
const onChangePage = (newPage: number) => {
scrollToTop();
navigateToSearchUrl({ type: activeType, query, page: newPage, filters, history });
navigateToSearchUrl({ type: activeType, query, page: newPage, filters, history, unionType });
};
/**
@ -139,6 +148,7 @@ export const SearchPage = () => {
return (
<>
<SearchResults
unionType={unionType}
entityFilters={entityFilters}
filtersWithoutEntities={filtersWithoutEntities}
callSearchOnVariables={callSearchOnVariables}
@ -150,6 +160,7 @@ export const SearchPage = () => {
selectedFilters={filters}
loading={loading}
onChangeFilters={onChangeFilters}
onChangeUnionType={onChangeUnionType}
onChangePage={onChangePage}
numResultsPerPage={numResultsPerPage}
setNumResultsPerPage={setNumResultsPerPage}

53
datahub-web-react/src/app/search/SearchResults.tsx
View File

@ -10,7 +10,6 @@ import {
MatchedField,
SearchAcrossEntitiesInput,
} from '../../types.generated';
import { SearchFilters } from './SearchFilters';
import { SearchCfg } from '../../conf';
import { SearchResultsRecommendations } from './SearchResultsRecommendations';
import { useGetAuthenticatedUser } from '../useGetAuthenticatedUser';
@ -23,6 +22,8 @@ import { isListSubset } from '../entity/shared/utils';
import TabToolbar from '../entity/shared/components/styled/TabToolbar';
import { EntityAndType } from '../entity/shared/types';
import { ErrorSection } from '../shared/error/ErrorSection';
import { UnionType } from './utils/constants';
import { SearchFiltersSection } from './SearchFiltersSection';
const SearchBody = styled.div`
display: flex;
@ -30,14 +31,6 @@ const SearchBody = styled.div`
min-height: calc(100vh - 60px);
`;
const FiltersContainer = styled.div`
display: block;
max-width: 260px;
min-width: 260px;
border-right: 1px solid;
border-color: ${(props) => props.theme.styles['border-color-base']};
`;
const ResultContainer = styled.div`
flex: 1;
margin-bottom: 20px;
@ -61,25 +54,6 @@ const PaginationInfoContainer = styled.div`
align-items: center;
`;
const FiltersHeader = styled.div`
font-size: 14px;
font-weight: 600;
padding-left: 20px;
padding-right: 20px;
padding-bottom: 8px;
width: 100%;
height: 47px;
line-height: 47px;
border-bottom: 1px solid;
border-color: ${(props) => props.theme.styles['border-color-base']};
`;
const SearchFilterContainer = styled.div`
padding-top: 10px;
`;
const SearchResultsRecommendationsContainer = styled.div`
margin-top: 40px;
`;
@ -92,6 +66,7 @@ const StyledTabToolbar = styled(TabToolbar)`
const SearchMenuContainer = styled.div``;
interface Props {
unionType?: UnionType;
query: string;
page: number;
searchResponse?: {
@ -108,6 +83,7 @@ interface Props {
loading: boolean;
error: any;
onChangeFilters: (filters: Array<FacetFilterInput>) => void;
onChangeUnionType: (unionType: UnionType) => void;
onChangePage: (page: number) => void;
callSearchOnVariables: (variables: {
input: SearchAcrossEntitiesInput;
@ -125,6 +101,7 @@ interface Props {
}
export const SearchResults = ({
unionType = UnionType.AND,
query,
page,
searchResponse,
@ -132,6 +109,7 @@ export const SearchResults = ({
selectedFilters,
loading,
error,
onChangeUnionType,
onChangeFilters,
onChangePage,
callSearchOnVariables,
@ -161,17 +139,14 @@ export const SearchResults = ({
{loading && <Message type="loading" content="Loading..." style={{ marginTop: '10%' }} />}
<div>
<SearchBody>
<FiltersContainer>
<FiltersHeader>Filter</FiltersHeader>
<SearchFilterContainer>
<SearchFilters
loading={loading}
facets={filters || []}
selectedFilters={selectedFilters}
onFilterSelect={(newFilters) => onChangeFilters(newFilters)}
/>
</SearchFilterContainer>
</FiltersContainer>
<SearchFiltersSection
filters={filters}
selectedFilters={selectedFilters}
unionType={unionType}
loading={loading}
onChangeFilters={onChangeFilters}
onChangeUnionType={onChangeUnionType}
/>
<ResultContainer>
<PaginationInfoContainer>
<>

11
datahub-web-react/src/app/search/SearchFilter.tsx → datahub-web-react/src/app/search/SimpleSearchFilter.tsx
View File

@ -5,7 +5,7 @@ import * as React from 'react';
import { useState } from 'react';
import styled from 'styled-components';
import { FacetMetadata } from '../../types.generated';
import { FacetFilterInput, FacetMetadata } from '../../types.generated';
import { SearchFilterLabel } from './SearchFilterLabel';
import { TRUNCATED_FILTER_LENGTH } from './utils/constants';
@ -17,10 +17,7 @@ const isGraphDegreeFilter = (field: string) => {
type Props = {
facet: FacetMetadata;
selectedFilters: Array<{
field: string;
value: string;
}>;
selectedFilters: Array<FacetFilterInput>;
onFilterSelect: (selected: boolean, field: string, value: string) => void;
defaultDisplayFilters: boolean;
};
@ -57,12 +54,12 @@ const StyledDownOutlined = styled(DownOutlined)`
font-size: 10px;
`;
export const SearchFilter = ({ facet, selectedFilters, onFilterSelect, defaultDisplayFilters }: Props) => {
export const SimpleSearchFilter = ({ facet, selectedFilters, onFilterSelect, defaultDisplayFilters }: Props) => {
const [areFiltersVisible, setAreFiltersVisible] = useState(defaultDisplayFilters);
const [expanded, setExpanded] = useState(false);
const isFacetSelected = (field, value) => {
return selectedFilters.find((f) => f.field === field && f.value === value) !== undefined;
return selectedFilters.find((f) => f.field === field && f.values.includes(value)) !== undefined;
};
// Aggregations filtered for count > 0 or selected = true

35
datahub-web-react/src/app/search/SearchFilters.tsx → datahub-web-react/src/app/search/SimpleSearchFilters.tsx
View File

@ -1,8 +1,8 @@
import * as React from 'react';
import styled from 'styled-components';
import { useEffect, useState } from 'react';
import { FacetMetadata } from '../../types.generated';
import { SearchFilter } from './SearchFilter';
import { FacetFilterInput, FacetMetadata } from '../../types.generated';
import { SimpleSearchFilter } from './SimpleSearchFilter';
const TOP_FILTERS = ['degree', 'entity', 'tags', 'glossaryTerms', 'domains', 'owners'];
@ -24,26 +24,15 @@ export const SearchFilterWrapper = styled.div`
interface Props {
facets: Array<FacetMetadata>;
selectedFilters: Array<{
field: string;
value: string;
}>;
onFilterSelect: (
newFilters: Array<{
field: string;
value: string;
}>,
) => void;
selectedFilters: Array<FacetFilterInput>;
onFilterSelect: (newFilters: Array<FacetFilterInput>) => void;
loading: boolean;
}
export const SearchFilters = ({ facets, selectedFilters, onFilterSelect, loading }: Props) => {
export const SimpleSearchFilters = ({ facets, selectedFilters, onFilterSelect, loading }: Props) => {
const [cachedProps, setCachedProps] = useState<{
facets: Array<FacetMetadata>;
selectedFilters: Array<{
field: string;
value: string;
}>;
selectedFilters: Array<FacetFilterInput>;
}>({
facets,
selectedFilters,
@ -58,8 +47,14 @@ export const SearchFilters = ({ facets, selectedFilters, onFilterSelect, loading
const onFilterSelectAndSetCache = (selected: boolean, field: string, value: string) => {
const newFilters = selected
? [...selectedFilters, { field, value }]
: selectedFilters.filter((filter) => filter.field !== field || filter.value !== value);
? [...selectedFilters, { field, values: [value] }]
: selectedFilters
.map((filter) =>
filter.field === field
? { ...filter, values: filter.values.filter((val) => val !== value) }
: filter,
)
.filter((filter) => filter.field !== field || !(filter.values.length === 0));
setCachedProps({ ...cachedProps, selectedFilters: newFilters });
onFilterSelect(newFilters);
};
@ -73,7 +68,7 @@ export const SearchFilters = ({ facets, selectedFilters, onFilterSelect, loading
return (
<SearchFilterWrapper>
{sortedFacets.map((facet) => (
<SearchFilter
<SimpleSearchFilter
key={`${facet.displayName}-${facet.field}`}
facet={facet}
selectedFilters={cachedProps.selectedFilters}

1
datahub-web-react/src/app/search/utils/constants.ts
View File

@ -39,6 +39,7 @@ export const FIELD_TO_LABEL = {
container: 'Container',
typeNames: 'Subtype',
origin: 'Environment',
degree: 'Degree',
};
export const FIELDS_THAT_USE_CONTAINS_OPERATOR = ['description', 'fieldDescriptions'];

10
datahub-web-react/src/app/search/utils/filtersToGraphqlParams.ts
View File

@ -1,10 +0,0 @@
import { FacetFilterInput } from '../../../types.generated';
export function filtersToGraphqlParams(filters: Array<FacetFilterInput>): Array<FacetFilterInput> {
return Object.entries(
filters.reduce((acc, filter) => {
acc[filter.field] = [...(acc[filter.field] || []), filter.value];
return acc;
}, {} as Record<string, string[]>),
).map(([field, values]) => ({ field, value: values.join(',') } as FacetFilterInput));
}

40
datahub-web-react/src/app/search/utils/filtersToQueryStringParams.ts
View File

@ -1,14 +1,36 @@
import { FacetFilterInput } from '../../../types.generated';
import { FacetFilterInput, FilterOperator } from '../../../types.generated';
import { encodeComma } from '../../entity/shared/utils';
import { FILTER_URL_PREFIX } from './constants';
import { DEGREE_FILTER, FILTER_URL_PREFIX } from './constants';
export const URL_PARAM_SEPARATOR = '___';
// In the checkbox-based filter view, usually, selecting two facets ANDs them together.
// E.g., if you select the checkbox for tagA and tagB, that means "has tagA AND tagB"
// we need to special case `degree` filter since it is a OR grouping vs the others which are ANDS by default
function reduceFiltersToCombineDegreeFilters(acc: FacetFilterInput[], filter: FacetFilterInput) {
// if we see a `degree` filter and we already have one, combine it with the other degree filter
if (filter.field === DEGREE_FILTER && acc.filter((f) => f.field === DEGREE_FILTER).length > 0) {
// instead of appending this new degree filter, combine it with the previous one and continue
return acc.map((f) =>
f.field === DEGREE_FILTER ? { ...f, values: [...f.values, ...filter.values] } : f,
) as FacetFilterInput[];
}
return [...acc, filter] as FacetFilterInput[];
}
// we need to reformat our list of filters into a dict
function reduceFiltersIntoQueryStringDict(acc, filter, idx) {
acc[
`${FILTER_URL_PREFIX}${filter.field}${URL_PARAM_SEPARATOR}${String(!!filter.negated)}${URL_PARAM_SEPARATOR}${
filter.condition || FilterOperator.Equal
}${URL_PARAM_SEPARATOR}${idx}`
] = [...filter.values.map((value) => encodeComma(value))];
return acc;
}
// transform filters from [{ filter, value }, { filter, value }] to { filter: [value, value ] } that QueryString can parse
export default function filtersToQueryStringParams(filters: Array<FacetFilterInput> = []) {
return filters.reduce((acc, filter) => {
acc[`${FILTER_URL_PREFIX}${filter.field}`] = [
...(acc[`${FILTER_URL_PREFIX}${filter.field}`] || []),
encodeComma(filter.value),
];
return acc;
}, {} as Record<string, string[]>);
return filters
.reduce(reduceFiltersToCombineDegreeFilters, [])
.reduce(reduceFiltersIntoQueryStringDict, {} as Record<string, string[]>);
}

20
datahub-web-react/src/app/search/utils/generateOrFilters.ts Normal file
View File

@ -0,0 +1,20 @@
import { FacetFilterInput, OrFilter } from '../../../types.generated';
import { UnionType } from './constants';
export function generateOrFilters(unionType: UnionType, filters: FacetFilterInput[]): OrFilter[] {
if ((filters?.length || 0) === 0) {
return [];
}
if (unionType === UnionType.OR) {
return filters.map((filter) => ({
and: [filter],
}));
}
return [
{
and: filters,
},
];
}

12
datahub-web-react/src/app/search/utils/hasAdvancedFilters.ts Normal file
View File

@ -0,0 +1,12 @@
import { FacetFilterInput } from '../../../types.generated';
import { ADVANCED_SEARCH_ONLY_FILTERS, UnionType } from './constants';
// utility method that looks at the set of filters and determines if the filters can be represented by simple search
export const hasAdvancedFilters = (filters: FacetFilterInput[], unionType: UnionType) => {
return (
filters.filter(
(filter) =>
ADVANCED_SEARCH_ONLY_FILTERS.indexOf(filter.field) >= 0 || filter.negated || unionType === UnionType.OR,
).length > 0
);
};

36
datahub-web-react/src/app/search/utils/navigateToSearchUrl.ts
View File

@ -4,12 +4,14 @@ import { RouteComponentProps } from 'react-router-dom';
import filtersToQueryStringParams from './filtersToQueryStringParams';
import { EntityType, FacetFilterInput } from '../../../types.generated';
import { PageRoutes } from '../../../conf/Global';
import { UnionType } from './constants';
export const navigateToSearchUrl = ({
type: newType,
query: newQuery,
page: newPage = 1,
filters: newFilters,
unionType = UnionType.AND,
history,
}: {
type?: EntityType;
@ -17,10 +19,11 @@ export const navigateToSearchUrl = ({
page?: number;
filters?: Array<FacetFilterInput>;
history: RouteComponentProps['history'];
unionType?: UnionType;
}) => {
const constructedFilters = newFilters || [];
if (newType) {
constructedFilters.push({ field: 'entity', value: newType });
constructedFilters.push({ field: 'entity', values: [newType] });
}
const search = QueryString.stringify(
@ -28,6 +31,7 @@ export const navigateToSearchUrl = ({
...filtersToQueryStringParams(constructedFilters),
query: encodeURIComponent(newQuery || ''),
page: newPage,
unionType,
},
{ arrayFormat: 'comma' },
);
@ -37,33 +41,3 @@ export const navigateToSearchUrl = ({
search,
});
};
export const navigateToSearchLineageUrl = ({
entityUrl,
query: newQuery,
page: newPage = 1,
filters: newFilters,
history,
}: {
entityUrl: string;
query?: string;
page?: number;
filters?: Array<FacetFilterInput>;
history: RouteComponentProps['history'];
}) => {
const constructedFilters = newFilters || [];
const search = QueryString.stringify(
{
...filtersToQueryStringParams(constructedFilters),
query: encodeURIComponent(newQuery || ''),
page: newPage,
},
{ arrayFormat: 'comma' },
);
history.push({
pathname: entityUrl,
search,
});
};

34
datahub-web-react/src/app/search/utils/useFilters.ts
View File

@ -2,27 +2,37 @@ import { useMemo } from 'react';
import * as QueryString from 'query-string';
import { FILTER_URL_PREFIX } from './constants';
import { FacetFilterInput } from '../../../types.generated';
import { FacetFilterInput, FilterOperator } from '../../../types.generated';
import { decodeComma } from '../../entity/shared/utils';
import { URL_PARAM_SEPARATOR } from './filtersToQueryStringParams';
export default function useFilters(params: QueryString.ParsedQuery<string>): Array<FacetFilterInput> {
return useMemo(
() =>
// get all query params
return useMemo(() => {
return (
Object.entries(params)
// select only the ones with the `filter_` prefix
.filter(([key, _]) => key.indexOf(FILTER_URL_PREFIX) >= 0)
// transform the filters currently in format [key, [value1, value2]] to [{key: key, value: value1}, { key: key, value: value2}] format that graphql expects
.flatMap(([key, value]) => {
.map(([key, value]) => {
// remove the `filter_` prefix
const field = key.replace(FILTER_URL_PREFIX, '');
if (!value) return [];
const fieldIndex = key.replace(FILTER_URL_PREFIX, '');
const fieldParts = fieldIndex.split(URL_PARAM_SEPARATOR);
const field = fieldParts[0];
const negated = fieldParts[1] === 'true';
const condition = fieldParts[2] || FilterOperator.Equal;
if (!value) return null;
if (Array.isArray(value)) {
return value.map((distinctValue) => ({ field, value: decodeComma(distinctValue) }));
return {
field,
condition,
negated,
values: value.map((distinctValue) => decodeComma(distinctValue)),
};
}
return [{ field, value: decodeComma(value) }];
}),
[params],
);
return { field, condition, values: [decodeComma(value)], negated };
})
.filter((val) => !!val) as Array<FacetFilterInput>
);
}, [params]);
}

2
datahub-web-react/src/app/settings/AccessTokens.tsx
View File

@ -93,7 +93,7 @@ export const AccessTokens = () => {
const filters: Array<FacetFilterInput> = [
{
field: 'ownerUrn',
value: currentUserUrn,
values: [currentUserUrn],
},
];

3
datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx
View File

@ -136,7 +136,7 @@ export default function EditTagTermsModal({
entity.type === EntityType.Tag ? (entity as Tag).name : entityRegistry.getDisplayName(entity.type, entity);
const tagOrTermComponent = <TagTermLabel entity={entity} />;
return (
<Select.Option value={entity.urn} key={entity.urn} name={displayName}>
<Select.Option data-testid="tag-term-option" value={entity.urn} key={entity.urn} name={displayName}>
{tagOrTermComponent}
</Select.Option>
);
@ -431,6 +431,7 @@ export default function EditTagTermsModal({
>
<ClickOutside onClickOutside={() => setIsFocusedOnInput(false)}>
<TagSelect
data-testid="tag-term-modal-input"
autoFocus
defaultOpen
mode="multiple"

2
datahub-web-react/src/graphql/analytics.graphql
View File

@ -61,7 +61,7 @@ fragment analyticsChart on AnalyticsChart {
query
filters {
field
value
values
}
}
entityProfileParams {

2
datahub-web-react/src/graphql/recommendations.graphql
View File

@ -15,7 +15,7 @@ query listRecommendations($input: ListRecommendationsInput!) {
query
filters {
field
value
values
}
}
entityProfileParams {

48
docs/how/search.md
View File

@ -39,6 +39,41 @@ The filters sidebar sits on the left hand side of search results, and lets users
<img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/filters_highlighted.png" />
</p>
### Advanced Filters
Using the Advanced Filter view, you can apply more complex filters. To get there, click 'Advanced' in the top right of the filter panel:
<p align="center">
<img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/advanced_search/click_to_advanced_search_view.png"/>
</p>
#### Adding an Advanced Filter
Currently, Advanced Filters support filtering by Column Name, Container, Domain, Description (entity or column level), Tag (entity or column level), Glossary Term (entity or column level), Owner, Entity Type, Subtype, Environment and soft-deleted status.
To add a new filter, click the add filter menu, choose a filter type, and then fill in the values you want to filter by.
<p align="center">
<img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/advanced_search/click_add_advanced_filter.png"/>
</p>
#### Matching Any Advanced Filter
By default, all filters must be matched in order for a result to appear. For example, if you add a tag filter and a platform filter, all results will have the tag and the platform. You can set the results to match any filter instead. Click on `all filters` and select `any filter` from the drop-down menu.
<p align="center">
<img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/advanced_search/advanced_search_choose_matches_any.png"/>
</p>
#### Negating An Advanced Filter
After creating a filter, you can choose whether results should or should not match it. Change this by clicking the operation in the top right of the filter and selecting the negated operation.
<p align="center">
<img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/advanced_search/advanced_search_select_negated.png"/>
</p>
### Results
Search results appear ranked by their relevance. In self-hosted DataHub ranking is based on how closely the query matched textual fields of an asset and its metadata. In Managed DataHub, ranking is based on a combination of textual relevance, usage (queries / views), and change frequency.
@ -142,7 +177,8 @@ The order of the search results is based on the weight what Datahub gives them b
The sample queries here are non exhaustive. [The link here](https://demo.datahubproject.io/tag/urn:li:tag:Searchable) shows the current list of indexed fields for each entity inside Datahub. Click on the fields inside each entity and see which field has the tag ```Searchable```.
However, it does not tell you the specific attribute name to use for specialized searches. One way to do so is to inspect the ElasticSearch indices, for example:
```curl http://localhost:9200/_cat/indices``` returns all the ES indices in the ElasticSearch container.
`curl http://localhost:9200/_cat/indices` returns all the ES indices in the ElasticSearch container.
```
yellow open chartindex_v2_1643510690325 bQO_RSiCSUiKJYsmJClsew 1 1 2 0 8.5kb 8.5kb
yellow open mlmodelgroupindex_v2_1643510678529 OjIy0wb7RyKqLz3uTENRHQ 1 1 0 0 208b 208b
@ -176,11 +212,13 @@ yellow open system_metadata_service_v1 36spEDbDTdKgVl
yellow open schemafieldindex_v2_1643510684410 tZ1gC3haTReRLmpCxirVxQ 1 1 0 0 208b 208b
yellow open mlfeatureindex_v2_1643510680246 aQO5HF0mT62Znn-oIWBC8A 1 1 20 0 17.4kb 17.4kb
yellow open tagindex_v2_1643510684785 PfnUdCUORY2fnF3I3W7HwA 1 1 3 1 18.6kb 18.6kb
```
The index name will vary from instance to instance. Indexed information about Datasets can be found in:
```curl http://localhost:9200/datasetindex_v2_1643510688970/_search?=pretty```
```
The index name will vary from instance to instance. Indexed information about Datasets can be found in:
`curl http://localhost:9200/datasetindex_v2_1643510688970/_search?=pretty`
example information of a dataset:
example information of a dataset:
```
{
"_index" : "datasetindex_v2_1643510688970",

0
metadata-ingestion/ingest_schema.py Normal file
View File

2
metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java
View File

@ -182,7 +182,7 @@ public class LineageSearchService {
List<String> degreeFilter = conjunctiveCriterion.getAnd()
.stream()
.filter(criterion -> criterion.getField().equals(DEGREE_FILTER_INPUT))
.map(Criterion::getValue)
.flatMap(c -> c.getValues().stream())
.collect(Collectors.toList());
if (!degreeFilter.isEmpty()) {
Predicate<Integer> degreePredicate = convertFilterToPredicate(degreeFilter);

3
metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java
View File

@ -171,7 +171,8 @@ public class AllEntitiesSearchAggregator {
entry -> Pair.of(entry.getKey(), new AggregationMetadata()
.setName(entry.getValue().getName())
.setDisplayName(entry.getValue().getDisplayName(GetMode.NULL))
.setAggregations(entry.getValue().getAggregations())
.setAggregations(
entry.getValue().getAggregations())
.setFilterValues(
trimFilterValues(entry.getValue().getFilterValues()))
)

42
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
View File

@ -1,6 +1,5 @@
package com.linkedin.metadata.search.elasticsearch.query.request;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.template.DoubleMap;
@ -61,6 +60,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import static com.linkedin.metadata.search.utils.ESUtils.*;
import static com.linkedin.metadata.utils.SearchUtil.*;
@ -69,6 +69,8 @@ public class SearchRequestHandler {
private static final Map<EntitySpec, SearchRequestHandler> REQUEST_HANDLER_BY_ENTITY_NAME = new ConcurrentHashMap<>();
private static final String REMOVED = "removed";
private static final String URN_FILTER = "urn";
private static final int DEFAULT_MAX_TERM_BUCKET_SIZE = 20;
private final EntitySpec _entitySpec;
@ -133,7 +135,7 @@ public class SearchRequestHandler {
boolean removedInOrFilter = false;
if (filter != null) {
removedInOrFilter = filter.getOr().stream().anyMatch(
or -> or.getAnd().stream().anyMatch(criterion -> criterion.getField().equals(REMOVED))
or -> or.getAnd().stream().anyMatch(criterion -> criterion.getField().equals(REMOVED) || criterion.getField().equals(REMOVED + KEYWORD_SUFFIX))
);
}
// Filter out entities that are marked "removed" if and only if filter does not contain a criterion referencing it.
@ -404,8 +406,8 @@ public class SearchRequestHandler {
/**
* Injects the missing conjunctive filters into the aggregations list.
*/
private List<AggregationMetadata> addFiltersToAggregationMetadata(@Nonnull final List<AggregationMetadata> originalMetadata, @Nullable final Filter filter) {
if (filter == null) {
public List<AggregationMetadata> addFiltersToAggregationMetadata(@Nonnull final List<AggregationMetadata> originalMetadata, @Nullable final Filter filter) {
if (filter == null) {
return originalMetadata;
}
if (filter.hasOr()) {
@ -416,7 +418,7 @@ public class SearchRequestHandler {
return originalMetadata;
}
private void addOrFiltersToAggregationMetadata(@Nonnull final ConjunctiveCriterionArray or, @Nonnull final List<AggregationMetadata> originalMetadata) {
void addOrFiltersToAggregationMetadata(@Nonnull final ConjunctiveCriterionArray or, @Nonnull final List<AggregationMetadata> originalMetadata) {
for (ConjunctiveCriterion conjunction : or) {
// For each item in the conjunction, inject an empty aggregation if necessary
addCriteriaFiltersToAggregationMetadata(conjunction.getAnd(), originalMetadata);
@ -445,6 +447,12 @@ public class SearchRequestHandler {
return;
}
// We don't want to add urn filters to the aggregations we return as a sidecar to search results.
// They are automatically added by searchAcrossLineage and we dont need them to show up in the filter panel.
if (finalFacetField.equals(URN_FILTER)) {
return;
}
if (aggregationMetadataMap.containsKey(finalFacetField)) {
/*
* If we already have aggregations for the facet field, simply inject any missing values counts into the set.
@ -452,7 +460,11 @@ public class SearchRequestHandler {
* Elasticsearch.
*/
AggregationMetadata originalAggMetadata = aggregationMetadataMap.get(finalFacetField);
addMissingAggregationValueToAggregationMetadata(criterion.getValue(), originalAggMetadata);
if (criterion.hasValues()) {
criterion.getValues().stream().forEach(value -> addMissingAggregationValueToAggregationMetadata(value, originalAggMetadata));
} else {
addMissingAggregationValueToAggregationMetadata(criterion.getValue(), originalAggMetadata);
}
} else {
/*
* If we do not have ANY aggregation for the facet field, then inject a new aggregation metadata object for the
@ -463,14 +475,18 @@ public class SearchRequestHandler {
originalMetadata.add(buildAggregationMetadata(
finalFacetField,
_filtersToDisplayName.getOrDefault(finalFacetField, finalFacetField),
new LongMap(ImmutableMap.of(criterion.getValue(), 0L)),
new FilterValueArray(ImmutableList.of(createFilterValue(criterion.getValue(), 0L))))
new LongMap(criterion.getValues().stream().collect(Collectors.toMap(i -> i, i -> 0L))),
new FilterValueArray(criterion.getValues().stream().map(value -> createFilterValue(value, 0L)).collect(
Collectors.toList())))
);
}
}
private void addMissingAggregationValueToAggregationMetadata(@Nonnull final String value, @Nonnull final AggregationMetadata originalMetadata) {
if (originalMetadata.getAggregations().entrySet().stream().noneMatch(entry -> value.equals(entry.getKey()))) {
if (
originalMetadata.getAggregations().entrySet().stream().noneMatch(entry -> value.equals(entry.getKey()))
|| originalMetadata.getFilterValues().stream().noneMatch(entry -> entry.getValue().equals(value))
) {
// No aggregation found for filtered value -- inject one!
originalMetadata.getAggregations().put(value, 0L);
originalMetadata.getFilterValues().add(createFilterValue(value, 0L));
@ -489,12 +505,4 @@ public class SearchRequestHandler {
.setFilterValues(filterValues);
}
@Nullable
private String toFacetField(@Nonnull final String filterField) {
String trimmedField = filterField.replace(ESUtils.KEYWORD_SUFFIX, "");
if (_facetFields.contains(trimmedField)) {
return trimmedField;
}
return null;
}
}

58
metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
View File

@ -1,11 +1,14 @@
package com.linkedin.metadata.search.utils;
import com.google.common.collect.ImmutableSet;
import com.linkedin.metadata.query.filter.Condition;
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
import com.linkedin.metadata.query.filter.Criterion;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.query.filter.SortCriterion;
import java.util.Arrays;
import java.util.Optional;
import java.util.Set;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import lombok.extern.slf4j.Slf4j;
@ -28,6 +31,18 @@ public class ESUtils {
public static final String KEYWORD_SUFFIX = ".keyword";
public static final int MAX_RESULT_SIZE = 10000;
// we use this to make sure we filter for editable & non-editable fields
public static final String[][] EDITABLE_FIELD_TO_QUERY_PAIRS = {
{"fieldGlossaryTags", "editedFieldGlossaryTags"},
{"fieldGlossaryTerms", "editedFieldGlossaryTerms"},
{"fieldDescriptions", "editedFieldDescriptions"},
{"description", "editedDescription"},
};
public static final Set<String> BOOLEAN_FIELDS = ImmutableSet.of(
"removed"
);
/*
* Refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/regexp-syntax.html for list of reserved
* characters in an Elasticsearch regular expression.
@ -76,7 +91,11 @@ public class ESUtils {
conjunctiveCriterion.getAnd().forEach(criterion -> {
if (!criterion.getValue().trim().isEmpty() || criterion.hasValues()
|| criterion.getCondition() == Condition.IS_NULL) {
andQueryBuilder.must(getQueryBuilderFromCriterion(criterion));
if (!criterion.isNegated()) {
andQueryBuilder.must(getQueryBuilderFromCriterion(criterion));
} else {
andQueryBuilder.mustNot(getQueryBuilderFromCriterion(criterion));
}
}
});
return andQueryBuilder;
@ -107,12 +126,42 @@ public class ESUtils {
*/
@Nonnull
public static QueryBuilder getQueryBuilderFromCriterion(@Nonnull Criterion criterion) {
String fieldName = toFacetField(criterion.getField());
Optional<String[]> pairMatch = Arrays.stream(EDITABLE_FIELD_TO_QUERY_PAIRS)
.filter(pair -> Arrays.stream(pair).anyMatch(pairValue -> pairValue.equals(fieldName)))
.findFirst();
if (pairMatch.isPresent()) {
final BoolQueryBuilder orQueryBuilder = new BoolQueryBuilder();
String[] pairMatchValue = pairMatch.get();
for (String field: pairMatchValue) {
Criterion criterionToQuery = new Criterion();
criterionToQuery.setCondition(criterion.getCondition());
criterionToQuery.setNegated(criterion.isNegated());
criterionToQuery.setValue(criterion.getValue());
criterionToQuery.setField(field + KEYWORD_SUFFIX);
orQueryBuilder.should(getQueryBuilderFromCriterionForSingleField(criterionToQuery));
}
return orQueryBuilder;
}
return getQueryBuilderFromCriterionForSingleField(criterion);
}
@Nonnull
public static QueryBuilder getQueryBuilderFromCriterionForSingleField(@Nonnull Criterion criterion) {
final Condition condition = criterion.getCondition();
String fieldName = toFacetField(criterion.getField());
if (condition == Condition.EQUAL) {
// If values is set, use terms query to match one of the values
if (!criterion.getValues().isEmpty()) {
if (BOOLEAN_FIELDS.contains(fieldName) && criterion.getValues().size() == 1) {
return QueryBuilders.termQuery(fieldName, Boolean.parseBoolean(criterion.getValues().get(0)));
}
return QueryBuilders.termsQuery(criterion.getField(), criterion.getValues());
}
// TODO(https://github.com/datahub-project/datahub-gma/issues/51): support multiple values a field can take without using
// delimiters like comma. This is a hack to support equals with URN that has a comma in it.
if (isUrn(criterion.getValue())) {
@ -185,4 +234,9 @@ public class ESUtils {
}
return input;
}
}
@Nullable
public static String toFacetField(@Nonnull final String filterField) {
return filterField.replace(ESUtils.KEYWORD_SUFFIX, "");
}
}

3
metadata-io/src/main/java/com/linkedin/metadata/search/utils/QueryUtils.java
View File

@ -3,6 +3,7 @@ package com.linkedin.metadata.search.utils;
import com.datahub.util.ModelUtils;
import com.google.common.collect.ImmutableList;
import com.linkedin.data.template.RecordTemplate;
import com.linkedin.data.template.StringArray;
import com.linkedin.metadata.aspect.AspectVersion;
import com.linkedin.metadata.dao.BaseReadDAO;
import com.linkedin.metadata.query.filter.Condition;
@ -39,7 +40,7 @@ public class QueryUtils {
// Creates new Criterion with field, value and condition.
@Nonnull
public static Criterion newCriterion(@Nonnull String field, @Nonnull String value, @Nonnull Condition condition) {
return new Criterion().setField(field).setValue(value).setCondition(condition);
return new Criterion().setField(field).setValue(value).setValues(new StringArray(ImmutableList.of(value))).setCondition(condition);
}
// Creates new Filter from a map of Criteria by removing null-valued Criteria and using EQUAL condition (default).

3
metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java
View File

@ -137,6 +137,7 @@ public class SearchUtils {
Stream.concat(one.getAggregations().entrySet().stream(), two.getAggregations().entrySet().stream())
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, Long::sum));
return one.clone()
.setDisplayName(two.getDisplayName() != two.getName() ? two.getDisplayName() : one.getDisplayName())
.setAggregations(new LongMap(mergedMap))
.setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(mergedMap)));
}
@ -153,4 +154,4 @@ public class SearchUtils {
new UrnArray(searchResult.getEntities().stream().map(SearchEntity::getEntity).collect(Collectors.toList())));
return listResult;
}
}
}

206
metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java
View File

@ -6,9 +6,16 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.ImmutableList;
import com.linkedin.common.urn.TestEntityUrn;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.template.StringArray;
import com.linkedin.metadata.ElasticTestUtils;
import com.linkedin.metadata.models.registry.EntityRegistry;
import com.linkedin.metadata.models.registry.SnapshotEntityRegistry;
import com.linkedin.metadata.query.filter.Condition;
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
import com.linkedin.metadata.query.filter.Criterion;
import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.search.aggregator.AllEntitiesSearchAggregator;
import com.linkedin.metadata.search.cache.CachingAllEntitiesSearchAggregator;
import com.linkedin.metadata.search.cache.EntityDocCountCache;
@ -160,4 +167,203 @@ public class SearchServiceTest {
searchResult = _searchService.searchAcrossEntities(ImmutableList.of(), "test", null, null, 0, 10, null);
assertEquals(searchResult.getNumEntities().intValue(), 0);
}
@Test
public void testAdvancedSearchOr() throws Exception {
final Criterion filterCriterion = new Criterion()
.setField("platform")
.setCondition(Condition.EQUAL)
.setValue("hive")
.setValues(new StringArray(ImmutableList.of("hive")));
final Criterion subtypeCriterion = new Criterion()
.setField("subtypes")
.setCondition(Condition.EQUAL)
.setValue("")
.setValues(new StringArray(ImmutableList.of("view")));
final Filter filterWithCondition = new Filter().setOr(
new ConjunctiveCriterionArray(
new ConjunctiveCriterion().setAnd(
new CriterionArray(ImmutableList.of(filterCriterion))),
new ConjunctiveCriterion().setAnd(
new CriterionArray(ImmutableList.of(subtypeCriterion)))
));
SearchResult searchResult =
_searchService.searchAcrossEntities(ImmutableList.of(ENTITY_NAME), "test", filterWithCondition, null, 0, 10, null);
assertEquals(searchResult.getNumEntities().intValue(), 0);
clearCache();
Urn urn = new TestEntityUrn("test", "testUrn", "VALUE_1");
ObjectNode document = JsonNodeFactory.instance.objectNode();
document.set("urn", JsonNodeFactory.instance.textNode(urn.toString()));
document.set("keyPart1", JsonNodeFactory.instance.textNode("test"));
document.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride"));
document.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c"));
document.set("subtypes", JsonNodeFactory.instance.textNode("view"));
document.set("platform", JsonNodeFactory.instance.textNode("snowflake"));
_elasticSearchService.upsertDocument(ENTITY_NAME, document.toString(), urn.toString());
Urn urn2 = new TestEntityUrn("test", "testUrn", "VALUE_2");
ObjectNode document2 = JsonNodeFactory.instance.objectNode();
document2.set("urn", JsonNodeFactory.instance.textNode(urn2.toString()));
document2.set("keyPart1", JsonNodeFactory.instance.textNode("test"));
document2.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride"));
document2.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c"));
document2.set("subtypes", JsonNodeFactory.instance.textNode("table"));
document2.set("platform", JsonNodeFactory.instance.textNode("hive"));
_elasticSearchService.upsertDocument(ENTITY_NAME, document2.toString(), urn2.toString());
Urn urn3 = new TestEntityUrn("test", "testUrn", "VALUE_3");
ObjectNode document3 = JsonNodeFactory.instance.objectNode();
document3.set("urn", JsonNodeFactory.instance.textNode(urn3.toString()));
document3.set("keyPart1", JsonNodeFactory.instance.textNode("test"));
document3.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride"));
document3.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c"));
document3.set("subtypes", JsonNodeFactory.instance.textNode("table"));
document3.set("platform", JsonNodeFactory.instance.textNode("snowflake"));
_elasticSearchService.upsertDocument(ENTITY_NAME, document3.toString(), urn3.toString());
syncAfterWrite(_searchClient);
searchResult = _searchService.searchAcrossEntities(ImmutableList.of(), "test", filterWithCondition, null, 0, 10, null);
assertEquals(searchResult.getNumEntities().intValue(), 2);
assertEquals(searchResult.getEntities().get(0).getEntity(), urn);
assertEquals(searchResult.getEntities().get(1).getEntity(), urn2);
clearCache();
}
@Test
public void testAdvancedSearchSoftDelete() throws Exception {
final Criterion filterCriterion = new Criterion()
.setField("platform")
.setCondition(Condition.EQUAL)
.setValue("hive")
.setValues(new StringArray(ImmutableList.of("hive")));
final Criterion removedCriterion = new Criterion()
.setField("removed")
.setCondition(Condition.EQUAL)
.setValue("")
.setValues(new StringArray(ImmutableList.of("true")));
final Filter filterWithCondition = new Filter().setOr(
new ConjunctiveCriterionArray(
new ConjunctiveCriterion().setAnd(
new CriterionArray(ImmutableList.of(filterCriterion, removedCriterion)))
));
SearchResult searchResult =
_searchService.searchAcrossEntities(ImmutableList.of(ENTITY_NAME), "test", filterWithCondition, null, 0, 10, null);
assertEquals(searchResult.getNumEntities().intValue(), 0);
clearCache();
Urn urn = new TestEntityUrn("test", "testUrn", "VALUE_1");
ObjectNode document = JsonNodeFactory.instance.objectNode();
document.set("urn", JsonNodeFactory.instance.textNode(urn.toString()));
document.set("keyPart1", JsonNodeFactory.instance.textNode("test"));
document.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride"));
document.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c"));
document.set("subtypes", JsonNodeFactory.instance.textNode("view"));
document.set("platform", JsonNodeFactory.instance.textNode("hive"));
document.set("removed", JsonNodeFactory.instance.booleanNode(true));
_elasticSearchService.upsertDocument(ENTITY_NAME, document.toString(), urn.toString());
Urn urn2 = new TestEntityUrn("test", "testUrn", "VALUE_2");
ObjectNode document2 = JsonNodeFactory.instance.objectNode();
document2.set("urn", JsonNodeFactory.instance.textNode(urn2.toString()));
document2.set("keyPart1", JsonNodeFactory.instance.textNode("test"));
document2.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride"));
document2.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c"));
document2.set("subtypes", JsonNodeFactory.instance.textNode("table"));
document2.set("platform", JsonNodeFactory.instance.textNode("hive"));
document.set("removed", JsonNodeFactory.instance.booleanNode(false));
_elasticSearchService.upsertDocument(ENTITY_NAME, document2.toString(), urn2.toString());
Urn urn3 = new TestEntityUrn("test", "testUrn", "VALUE_3");
ObjectNode document3 = JsonNodeFactory.instance.objectNode();
document3.set("urn", JsonNodeFactory.instance.textNode(urn3.toString()));
document3.set("keyPart1", JsonNodeFactory.instance.textNode("test"));
document3.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride"));
document3.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c"));
document3.set("subtypes", JsonNodeFactory.instance.textNode("table"));
document3.set("platform", JsonNodeFactory.instance.textNode("snowflake"));
document.set("removed", JsonNodeFactory.instance.booleanNode(false));
_elasticSearchService.upsertDocument(ENTITY_NAME, document3.toString(), urn3.toString());
syncAfterWrite(_searchClient);
searchResult = _searchService.searchAcrossEntities(ImmutableList.of(), "test", filterWithCondition, null, 0, 10, null);
assertEquals(searchResult.getNumEntities().intValue(), 1);
assertEquals(searchResult.getEntities().get(0).getEntity(), urn);
clearCache();
}
@Test
public void testAdvancedSearchNegated() throws Exception {
final Criterion filterCriterion = new Criterion()
.setField("platform")
.setCondition(Condition.EQUAL)
.setValue("hive")
.setNegated(true)
.setValues(new StringArray(ImmutableList.of("hive")));
final Filter filterWithCondition = new Filter().setOr(
new ConjunctiveCriterionArray(
new ConjunctiveCriterion().setAnd(
new CriterionArray(ImmutableList.of(filterCriterion)))
));
SearchResult searchResult =
_searchService.searchAcrossEntities(ImmutableList.of(ENTITY_NAME), "test", filterWithCondition, null, 0, 10, null);
assertEquals(searchResult.getNumEntities().intValue(), 0);
clearCache();
Urn urn = new TestEntityUrn("test", "testUrn", "VALUE_1");
ObjectNode document = JsonNodeFactory.instance.objectNode();
document.set("urn", JsonNodeFactory.instance.textNode(urn.toString()));
document.set("keyPart1", JsonNodeFactory.instance.textNode("test"));
document.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride"));
document.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c"));
document.set("subtypes", JsonNodeFactory.instance.textNode("view"));
document.set("platform", JsonNodeFactory.instance.textNode("hive"));
document.set("removed", JsonNodeFactory.instance.booleanNode(true));
_elasticSearchService.upsertDocument(ENTITY_NAME, document.toString(), urn.toString());
Urn urn2 = new TestEntityUrn("test", "testUrn", "VALUE_2");
ObjectNode document2 = JsonNodeFactory.instance.objectNode();
document2.set("urn", JsonNodeFactory.instance.textNode(urn2.toString()));
document2.set("keyPart1", JsonNodeFactory.instance.textNode("test"));
document2.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride"));
document2.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c"));
document2.set("subtypes", JsonNodeFactory.instance.textNode("table"));
document2.set("platform", JsonNodeFactory.instance.textNode("hive"));
document.set("removed", JsonNodeFactory.instance.booleanNode(false));
_elasticSearchService.upsertDocument(ENTITY_NAME, document2.toString(), urn2.toString());
Urn urn3 = new TestEntityUrn("test", "testUrn", "VALUE_3");
ObjectNode document3 = JsonNodeFactory.instance.objectNode();
document3.set("urn", JsonNodeFactory.instance.textNode(urn3.toString()));
document3.set("keyPart1", JsonNodeFactory.instance.textNode("test"));
document3.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride"));
document3.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c"));
document3.set("subtypes", JsonNodeFactory.instance.textNode("table"));
document3.set("platform", JsonNodeFactory.instance.textNode("snowflake"));
document.set("removed", JsonNodeFactory.instance.booleanNode(false));
_elasticSearchService.upsertDocument(ENTITY_NAME, document3.toString(), urn3.toString());
syncAfterWrite(_searchClient);
searchResult = _searchService.searchAcrossEntities(ImmutableList.of(), "test", filterWithCondition, null, 0, 10, null);
assertEquals(searchResult.getNumEntities().intValue(), 1);
assertEquals(searchResult.getEntities().get(0).getEntity(), urn3);
clearCache();
}
}

7
metadata-models/src/main/pegasus/com/linkedin/metadata/query/filter/Criterion.pdl
View File

@ -25,4 +25,9 @@ record Criterion {
* The condition for the criterion, e.g. EQUAL, START_WITH
*/
condition: Condition = "EQUAL"
}
/**
* Whether the condition should be negated
*/
negated: boolean = false
}

5
metadata-service/restli-api/src/main/snapshot/com.linkedin.analytics.analytics.snapshot.json
View File

@ -72,6 +72,11 @@
},
"doc" : "The condition for the criterion, e.g. EQUAL, START_WITH",
"default" : "EQUAL"
}, {
"name" : "negated",
"type" : "boolean",
"doc" : "Whether the condition should be negated",
"default" : false
} ]
}
},

5
metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json
View File

@ -162,6 +162,11 @@
},
"doc" : "The condition for the criterion, e.g. EQUAL, START_WITH",
"default" : "EQUAL"
}, {
"name" : "negated",
"type" : "boolean",
"doc" : "Whether the condition should be negated",
"default" : false
} ]
}
},

5
metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json
View File

@ -5248,6 +5248,11 @@
"type" : "Condition",
"doc" : "The condition for the criterion, e.g. EQUAL, START_WITH",
"default" : "EQUAL"
}, {
"name" : "negated",
"type" : "boolean",
"doc" : "Whether the condition should be negated",
"default" : false
} ]
}
},

193
smoke-test/tests/cypress/cypress/integration/search/search.js
View File

@ -1,65 +1,192 @@
describe('search', () => {
it('can hit all entities search, see some results (testing this any more is tricky because it is cached for now)', () => {
describe("search", () => {
it("can hit all entities search, see some results (testing this any more is tricky because it is cached for now)", () => {
cy.login();
cy.visit('/');
cy.get('input[data-testid=search-input]').type('*{enter}');
cy.visit("/");
cy.get("input[data-testid=search-input]").type("*{enter}");
cy.wait(5000);
cy.contains('of 0 results').should('not.exist');
cy.contains(/of [0-9]+ results/);
cy.contains("of 0 results").should("not.exist");
cy.contains(/of [0-9]+ results/);
});
it('can hit all entities search with an impossible query and find 0 results', () => {
it("can hit all entities search with an impossible query and find 0 results", () => {
cy.login();
cy.visit('/');
cy.visit("/");
// random string that is unlikely to accidentally have a match
cy.get('input[data-testid=search-input]').type('zzzzzzzzzzzzzqqqqqqqqqqqqqzzzzzzqzqzqzqzq{enter}');
cy.get("input[data-testid=search-input]").type(
"zzzzzzzzzzzzzqqqqqqqqqqqqqzzzzzzqzqzqzqzq{enter}"
);
cy.wait(5000);
cy.contains('of 0 results');
cy.contains("of 0 results");
});
it('can search, find a result, and visit the dataset page', () => {
it("can search, find a result, and visit the dataset page", () => {
cy.login();
cy.visit('http://localhost:9002/search?filter_entity=DATASET&filter_tags=urn%3Ali%3Atag%3ACypress&page=1&query=users_created')
cy.contains('of 1 result');
cy.visit(
"/search?filter_entity=DATASET&filter_tags=urn%3Ali%3Atag%3ACypress&page=1&query=users_created"
);
cy.contains("of 1 result");
cy.contains('Cypress')
cy.contains("Cypress");
cy.contains('fct_cypress_users_created').click();
cy.contains("fct_cypress_users_created").click();
// platform
cy.contains('Hive');
cy.contains("Hive");
// entity type
cy.contains('Dataset');
cy.contains("Dataset");
// entity name
cy.contains('fct_cypress_users_created');
cy.contains("fct_cypress_users_created");
// column name
cy.contains('user_id');
cy.contains("user_id");
// column description
cy.contains('Id of the user');
cy.contains("Id of the user");
// table description
cy.contains('table containing all the users created on a single day');
cy.contains("table containing all the users created on a single day");
});
it('can search and get glossary term facets with proper labels', () => {
it("can search and get glossary term facets with proper labels", () => {
cy.login();
cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)');
cy.contains('cypress_logging_events');
cy.visit(
"/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"
);
cy.contains("cypress_logging_events");
cy.contains('Add Term').click();
cy.contains("Add Term").click();
cy.focused().type('CypressTerm');
cy.focused().type("CypressTerm");
cy.get('.ant-select-item-option-content').within(() => cy.contains('CypressTerm').click({force: true}));
cy.get(".ant-select-item-option-content").within(() =>
cy.contains("CypressTerm").click({ force: true })
);
cy.get('[data-testid="add-tag-term-from-modal-btn"]').click({force: true});
cy.get('[data-testid="add-tag-term-from-modal-btn"]').should('not.exist');
cy.get('[data-testid="add-tag-term-from-modal-btn"]').click({
force: true,
});
cy.get('[data-testid="add-tag-term-from-modal-btn"]').should("not.exist");
cy.contains('CypressTerm');
cy.visit('http://localhost:9002/search?query=cypress')
cy.contains('CypressTerm')
cy.contains("CypressTerm");
cy.visit("/search?query=cypress");
cy.contains("CypressTerm");
});
})
it("can search by a specific term using advanced search", () => {
cy.login();
cy.visit("/");
cy.get("input[data-testid=search-input]").type("*{enter}");
cy.wait(2000);
cy.contains("Advanced").click();
cy.contains("Add Filter").click();
cy.contains("Column Term").click({ force: true });
cy.get('[data-testid="tag-term-modal-input"]').type("CypressColumnInfo");
cy.wait(2000);
cy.get('[data-testid="tag-term-option"]').click({ force: true });
cy.get('[data-testid="add-tag-term-from-modal-btn"]').click({
force: true,
});
cy.wait(2000);
// has the term in editable metadata
cy.contains("SampleCypressHdfsDataset");
// has the term in non-editable metadata
cy.contains("cypress_logging_events");
cy.contains("of 2 results");
});
it("can search by AND-ing two concepts using advanced search", () => {
cy.login();
cy.visit("/");
cy.get("input[data-testid=search-input]").type("*{enter}");
cy.wait(2000);
cy.contains("Advanced").click();
cy.contains("Add Filter").click();
cy.contains("Column Term").click({ force: true });
cy.get('[data-testid="tag-term-modal-input"]').type("CypressColumnInfo");
cy.wait(2000);
cy.get('[data-testid="tag-term-option"]').click({ force: true });
cy.get('[data-testid="add-tag-term-from-modal-btn"]').click({
force: true,
});
cy.wait(2000);
cy.contains("Add Filter").click();
cy.get('[data-testid="adv-search-add-filter-description"]').click({
force: true,
});
cy.get('[data-testid="edit-text-input"]').type("log event");
cy.get('[data-testid="edit-text-done-btn"]').click({ force: true });
// has the term in non-editable metadata
cy.contains("cypress_logging_events");
});
it("can search by OR-ing two concepts using advanced search", () => {
cy.login();
cy.visit("/");
cy.get("input[data-testid=search-input]").type("*{enter}");
cy.wait(2000);
cy.contains("Advanced").click();
cy.contains("Add Filter").click();
cy.contains("Column Term").click({ force: true });
cy.get('[data-testid="tag-term-modal-input"]').type("CypressColumnInfo");
cy.wait(2000);
cy.get('[data-testid="tag-term-option"]').click({ force: true });
cy.get('[data-testid="add-tag-term-from-modal-btn"]').click({
force: true,
});
cy.wait(2000);
cy.contains("Add Filter").click();
cy.get('[data-testid="adv-search-add-filter-description"]').click({
force: true,
});
cy.get('[data-testid="edit-text-input"]').type("log event");
cy.get('[data-testid="edit-text-done-btn"]').click({ force: true });
// has the term in non-editable metadata
cy.contains("all filters").click();
cy.contains("any filter").click({ force: true });
cy.contains("cypress_logging_events");
cy.contains("fct_cypress_users_created_no_tag");
cy.contains("SampleCypressHdfsDataset");
});
});

43
smoke-test/tests/cypress/data.json
View File

@ -223,7 +223,8 @@
"editableSchemaFieldInfo": [
{
"fieldPath": "shipment_info",
"globalTags": { "tags": [{ "tag": "urn:li:tag:Legacy" }] }
"globalTags": { "tags": [{ "tag": "urn:li:tag:Legacy" }] },
"glossaryTerms": { "terms": [{ "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" }], "auditStamp": { "time": 0, "actor": "urn:li:corpuser:jdoe", "impersonator": null }}
}
]
}
@ -621,7 +622,8 @@
}
},
"nativeDataType": "boolean",
"recursive": false
"recursive": false,
"glossaryTerms": { "terms": [{ "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" }], "auditStamp": { "time": 0, "actor": "urn:li:corpuser:jdoe", "impersonator": null }}
},
{
"fieldPath": "timestamp",
@ -673,7 +675,7 @@
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"description": "table containing all the users created on a single day",
"description": "table containing all the users created on a single day. Creted from log events.",
"uri": null,
"tags": [],
"customProperties": {
@ -1820,5 +1822,40 @@
},
"proposedDelta": null,
"systemMetadata": null
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": {
"urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType",
"aspects": [
{
"com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": {
"definition": "a definition",
"parentNode": "urn:li:glossaryNode:CypressNode",
"sourceRef": "FIBO",
"termSource": "EXTERNAL",
"sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct",
"customProperties": {
"FQDN": "SavingAccount"
}
}
},
{
"com.linkedin.pegasus2avro.common.Ownership": {
"owners": [{
"owner": "urn:li:corpuser:jdoe",
"type": "DATAOWNER"
}],
"lastModified": {
"time": 1581407189000,
"actor": "urn:li:corpuser:jdoe"
}
}
}
]
}
},
"proposedDelta": null
}
]

10
smoke-test/tests/tokens/revokable_access_token_test.py
View File

@ -262,7 +262,7 @@ def test_non_admin_can_create_list_revoke_tokens(wait_for_healthchecks):
# User should be able to list his own token
res_data = listAccessTokens(
user_session, [{"field": "ownerUrn", "value": "urn:li:corpuser:user"}]
user_session, [{"field": "ownerUrn", "values": ["urn:li:corpuser:user"]}]
)
assert res_data
assert res_data["data"]
@ -289,7 +289,7 @@ def test_non_admin_can_create_list_revoke_tokens(wait_for_healthchecks):
# Using a normal account, check that all its tokens where removed.
res_data = listAccessTokens(
user_session, [{"field": "ownerUrn", "value": "urn:li:corpuser:user"}]
user_session, [{"field": "ownerUrn", "values": ["urn:li:corpuser:user"]}]
)
assert res_data
assert res_data["data"]
@ -331,7 +331,7 @@ def test_admin_can_manage_tokens_generated_by_other_user(wait_for_healthchecks):
user_session.cookies.clear()
admin_session = loginAs(admin_user, admin_pass)
res_data = listAccessTokens(
admin_session, [{"field": "ownerUrn", "value": "urn:li:corpuser:user"}]
admin_session, [{"field": "ownerUrn", "values": ["urn:li:corpuser:user"]}]
)
assert res_data
assert res_data["data"]
@ -362,7 +362,7 @@ def test_admin_can_manage_tokens_generated_by_other_user(wait_for_healthchecks):
user_session.cookies.clear()
user_session = loginAs("user", "user")
res_data = listAccessTokens(
user_session, [{"field": "ownerUrn", "value": "urn:li:corpuser:user"}]
user_session, [{"field": "ownerUrn", "values": ["urn:li:corpuser:user"]}]
)
assert res_data
assert res_data["data"]
@ -372,7 +372,7 @@ def test_admin_can_manage_tokens_generated_by_other_user(wait_for_healthchecks):
# Using the super account, check that all tokens where removed.
admin_session = loginAs(admin_user, admin_pass)
res_data = listAccessTokens(
admin_session, [{"field": "ownerUrn", "value": "urn:li:corpuser:user"}]
admin_session, [{"field": "ownerUrn", "values": ["urn:li:corpuser:user"]}]
)
assert res_data
assert res_data["data"]