yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-03 04:10:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(auto-complete) Pass in views to auto-complete endpoint for filtering (#7754)

Browse Source
This commit is contained in:
Chris Collins 2023-04-05 07:18:08 -04:00 committed by GitHub
parent e06117af66
commit 70e60847a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 217 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
View File

@ -688,7 +688,7 @@ public class GmsGraphQLEngine {
.dataFetcher("searchAcrossLineage", new SearchAcrossLineageResolver(this.entityClient))
.dataFetcher("scrollAcrossLineage", new ScrollAcrossLineageResolver(this.entityClient))
.dataFetcher("autoComplete", new AutoCompleteResolver(searchableTypes))
.dataFetcher("autoCompleteForMultiple", new AutoCompleteForMultipleResolver(searchableTypes))
.dataFetcher("autoCompleteForMultiple", new AutoCompleteForMultipleResolver(searchableTypes, this.viewService))
.dataFetcher("browse", new BrowseResolver(browsableTypes))
.dataFetcher("browsePaths", new BrowsePathsResolver(browsableTypes))
.dataFetcher("dataset", getResolver(datasetType))

40
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java
View File

@ -1,17 +1,24 @@
package com.linkedin.datahub.graphql.resolvers.search;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.exception.ValidationException;
import com.linkedin.datahub.graphql.generated.AutoCompleteMultipleInput;
import com.linkedin.datahub.graphql.generated.AutoCompleteMultipleResults;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper;
import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
import com.linkedin.datahub.graphql.types.SearchableEntityType;
import com.linkedin.metadata.service.ViewService;
import com.linkedin.view.DataHubViewInfo;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.ArrayList;
import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import org.slf4j.Logger;
@ -29,16 +36,19 @@ public class AutoCompleteForMultipleResolver implements DataFetcher<CompletableF
private static final Logger _logger = LoggerFactory.getLogger(AutoCompleteForMultipleResolver.class.getName());
private final Map<EntityType, SearchableEntityType<?, ?>> _typeToEntity;
private final ViewService _viewService;
public AutoCompleteForMultipleResolver(@Nonnull final List<SearchableEntityType<?, ?>> searchableEntities) {
public AutoCompleteForMultipleResolver(@Nonnull final List<SearchableEntityType<?, ?>> searchableEntities, @Nonnull final ViewService viewService) {
_typeToEntity = searchableEntities.stream().collect(Collectors.toMap(
SearchableEntityType::type,
entity -> entity
));
_viewService = viewService;
}
@Override
public CompletableFuture<AutoCompleteMultipleResults> get(DataFetchingEnvironment environment) {
final QueryContext context = environment.getContext();
final AutoCompleteMultipleInput input = bindArgument(environment.getArgument("input"), AutoCompleteMultipleInput.class);
if (isBlank(input.getQuery())) {
@ -47,14 +57,18 @@ public class AutoCompleteForMultipleResolver implements DataFetcher<CompletableF
}
// escape forward slash since it is a reserved character in Elasticsearch
final String sanitizedQuery = ResolverUtils.escapeForwardSlash(input.getQuery());
final DataHubViewInfo maybeResolvedView = (input.getViewUrn() != null)
? resolveView(_viewService, UrnUtils.getUrn(input.getViewUrn()), context.getAuthentication())
: null;
List<EntityType> types = input.getTypes();
List<EntityType> types = getEntityTypes(input.getTypes(), maybeResolvedView);
if (types != null && types.size() > 0) {
return AutocompleteUtils.batchGetAutocompleteResults(
types.stream().map(_typeToEntity::get).collect(Collectors.toList()),
sanitizedQuery,
input,
environment);
environment,
maybeResolvedView);
}
// By default, autocomplete only against the Default Set of Autocomplete entities
@ -62,6 +76,24 @@ public class AutoCompleteForMultipleResolver implements DataFetcher<CompletableF
AUTO_COMPLETE_ENTITY_TYPES.stream().map(_typeToEntity::get).collect(Collectors.toList()),
sanitizedQuery,
input,
environment);
environment,
maybeResolvedView);
}
/**
* Gets the intersection of provided input types and types on the view applied (if any)
*/
@Nullable
List<EntityType> getEntityTypes(final @Nullable List<EntityType> inputTypes, final @Nullable DataHubViewInfo maybeResolvedView) {
List<EntityType> types = inputTypes;
if (maybeResolvedView != null) {
List<EntityType> inputEntityTypes = types != null ? types : new ArrayList<>();
final List<String> inputEntityNames = inputEntityTypes.stream().map(EntityTypeMapper::getName).collect(Collectors.toList());
List<String> stringEntityTypes = SearchUtils.intersectEntityTypes(inputEntityNames, maybeResolvedView.getDefinition().getEntityTypes());
types = stringEntityTypes.stream().map(EntityTypeMapper::getType).collect(Collectors.toList());
}
return types;
}
}

12
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java
View File

@ -7,6 +7,7 @@ import com.linkedin.datahub.graphql.generated.AutoCompleteResults;
import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
import com.linkedin.datahub.graphql.types.SearchableEntityType;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.view.DataHubViewInfo;
import graphql.schema.DataFetchingEnvironment;
import java.util.ArrayList;
import java.util.Collections;
@ -16,6 +17,8 @@ import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
public class AutocompleteUtils {
private static final Logger _logger = LoggerFactory.getLogger(AutocompleteUtils.class.getName());
@ -28,17 +31,22 @@ public class AutocompleteUtils {
List<SearchableEntityType<?, ?>> entities,
String sanitizedQuery,
AutoCompleteMultipleInput input,
DataFetchingEnvironment environment
DataFetchingEnvironment environment,
@Nullable DataHubViewInfo view
) {
final int limit = input.getLimit() != null ? input.getLimit() : DEFAULT_LIMIT;
final List<CompletableFuture<AutoCompleteResultForEntity>> autoCompletesFuture = entities.stream().map(entity -> CompletableFuture.supplyAsync(() -> {
final Filter filter = ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters());
final Filter finalFilter = view != null
? SearchUtils.combineFilters(filter, view.getDefinition().getFilter())
: filter;
try {
final AutoCompleteResults searchResult = entity.autoComplete(
sanitizedQuery,
input.getField(),
filter,
finalFilter,
limit,
environment.getContext()
);

5
datahub-graphql-core/src/main/resources/search.graphql
View File

@ -718,6 +718,11 @@ input AutoCompleteMultipleInput {
A list of disjunctive criterion for the filter. (or operation to combine filters)
"""
orFilters: [AndFilterInput!]
"""
Optional - A View to apply when generating results
"""
viewUrn: String
}
"""

141
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolverTest.java
View File

@ -2,6 +2,9 @@ package com.linkedin.datahub.graphql.resolvers.search;
import com.datahub.authentication.Authentication;
import com.google.common.collect.ImmutableList;
import com.linkedin.common.AuditStamp;
import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.data.template.StringArray;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.exception.ValidationException;
@ -15,7 +18,15 @@ import com.linkedin.entity.client.EntityClient;
import com.linkedin.metadata.Constants;
import com.linkedin.metadata.query.AutoCompleteEntityArray;
import com.linkedin.metadata.query.AutoCompleteResult;
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
import com.linkedin.metadata.query.filter.Criterion;
import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.service.ViewService;
import com.linkedin.view.DataHubViewDefinition;
import com.linkedin.view.DataHubViewInfo;
import com.linkedin.view.DataHubViewType;
import graphql.schema.DataFetchingEnvironment;
import org.mockito.Mockito;
import org.testng.Assert;
@ -25,15 +36,21 @@ import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext;
public class AutoCompleteForMultipleResolverTest {
private static final Urn TEST_VIEW_URN = UrnUtils.getUrn("urn:li:dataHubView:test");
private static final Urn TEST_USER_URN = UrnUtils.getUrn("urn:li:corpuser:test");
private AutoCompleteForMultipleResolverTest() { }
public static void testAutoCompleteResolverSuccess(
EntityClient mockClient,
ViewService viewService,
String entityName,
EntityType entityType,
SearchableEntityType<?, ?> entity
SearchableEntityType<?, ?> entity,
Urn viewUrn,
Filter filter
) throws Exception {
final AutoCompleteForMultipleResolver resolver = new AutoCompleteForMultipleResolver(ImmutableList.of(entity));
final AutoCompleteForMultipleResolver resolver = new AutoCompleteForMultipleResolver(ImmutableList.of(entity), viewService);
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
QueryContext mockContext = getMockAllowContext();
@ -41,6 +58,9 @@ public class AutoCompleteForMultipleResolverTest {
input.setQuery("test");
input.setTypes(ImmutableList.of(entityType));
input.setLimit(10);
if (viewUrn != null) {
input.setViewUrn(viewUrn.toString());
}
Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input);
Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
@ -49,7 +69,7 @@ public class AutoCompleteForMultipleResolverTest {
mockClient,
entityName,
"test",
null,
filter,
10
);
}
@ -57,6 +77,7 @@ public class AutoCompleteForMultipleResolverTest {
// test our main entity types
@Test
public static void testAutoCompleteResolverSuccessForDifferentEntities() throws Exception {
ViewService viewService = initMockViewService(null, null);
// Daatasets
EntityClient mockClient = initMockEntityClient(
Constants.DATASET_ENTITY_NAME,
@ -68,7 +89,7 @@ public class AutoCompleteForMultipleResolverTest {
.setEntities(new AutoCompleteEntityArray())
.setSuggestions(new StringArray())
);
testAutoCompleteResolverSuccess(mockClient, Constants.DATASET_ENTITY_NAME, EntityType.DATASET, new DatasetType(mockClient));
testAutoCompleteResolverSuccess(mockClient, viewService, Constants.DATASET_ENTITY_NAME, EntityType.DATASET, new DatasetType(mockClient), null, null);
// Dashboards
mockClient = initMockEntityClient(
@ -81,7 +102,7 @@ public class AutoCompleteForMultipleResolverTest {
.setEntities(new AutoCompleteEntityArray())
.setSuggestions(new StringArray())
);
testAutoCompleteResolverSuccess(mockClient, Constants.DASHBOARD_ENTITY_NAME, EntityType.DASHBOARD, new DashboardType(mockClient));
testAutoCompleteResolverSuccess(mockClient, viewService, Constants.DASHBOARD_ENTITY_NAME, EntityType.DASHBOARD, new DashboardType(mockClient), null, null);
//DataFlows
mockClient = initMockEntityClient(
@ -94,13 +115,81 @@ public class AutoCompleteForMultipleResolverTest {
.setEntities(new AutoCompleteEntityArray())
.setSuggestions(new StringArray())
);
testAutoCompleteResolverSuccess(mockClient, Constants.DATA_FLOW_ENTITY_NAME, EntityType.DATA_FLOW, new DataFlowType(mockClient));
testAutoCompleteResolverSuccess(mockClient, viewService, Constants.DATA_FLOW_ENTITY_NAME, EntityType.DATA_FLOW, new DataFlowType(mockClient), null, null);
}
// test filters with a given view
@Test
public static void testAutoCompleteResolverWithViewFilter() throws Exception {
DataHubViewInfo viewInfo = createViewInfo(new StringArray());
ViewService viewService = initMockViewService(TEST_VIEW_URN, viewInfo);
EntityClient mockClient = initMockEntityClient(
Constants.DATASET_ENTITY_NAME,
"test",
null,
10,
new AutoCompleteResult()
.setQuery("test")
.setEntities(new AutoCompleteEntityArray())
.setSuggestions(new StringArray())
);
testAutoCompleteResolverSuccess(
mockClient,
viewService,
Constants.DATASET_ENTITY_NAME,
EntityType.DATASET,
new DatasetType(mockClient),
TEST_VIEW_URN,
viewInfo.getDefinition().getFilter()
);
}
// test entity type filters with a given view
@Test
public static void testAutoCompleteResolverWithViewEntityFilter() throws Exception {
// have view be a filter to only get dashboards
StringArray entityNames = new StringArray();
entityNames.add(Constants.DASHBOARD_ENTITY_NAME);
DataHubViewInfo viewInfo = createViewInfo(entityNames);
ViewService viewService = initMockViewService(TEST_VIEW_URN, viewInfo);
EntityClient mockClient = initMockEntityClient(
Constants.DASHBOARD_ENTITY_NAME,
"test",
null,
10,
new AutoCompleteResult()
.setQuery("test")
.setEntities(new AutoCompleteEntityArray())
.setSuggestions(new StringArray())
);
// ensure we do hit the entity client for dashboards since dashboards are in our view
testAutoCompleteResolverSuccess(
mockClient,
viewService,
Constants.DASHBOARD_ENTITY_NAME,
EntityType.DASHBOARD,
new DashboardType(mockClient),
TEST_VIEW_URN,
viewInfo.getDefinition().getFilter()
);
// if the view has only dashboards, we should not make an auto-complete request on other entity types
Mockito.verify(mockClient, Mockito.times(0))
.autoComplete(
Mockito.eq(Constants.DATASET_ENTITY_NAME),
Mockito.eq("test"),
Mockito.eq(viewInfo.getDefinition().getFilter()),
Mockito.eq(10),
Mockito.any(Authentication.class)
);
}
@Test
public static void testAutoCompleteResolverFailNoQuery() throws Exception {
EntityClient mockClient = Mockito.mock(EntityClient.class);
final AutoCompleteForMultipleResolver resolver = new AutoCompleteForMultipleResolver(ImmutableList.of(new DatasetType(mockClient)));
ViewService viewService = initMockViewService(null, null);
final AutoCompleteForMultipleResolver resolver = new AutoCompleteForMultipleResolver(ImmutableList.of(new DatasetType(mockClient)), viewService);
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
QueryContext mockContext = getMockAllowContext();
@ -132,6 +221,20 @@ public class AutoCompleteForMultipleResolverTest {
return client;
}
private static ViewService initMockViewService(
Urn viewUrn,
DataHubViewInfo viewInfo
) {
ViewService service = Mockito.mock(ViewService.class);
Mockito.when(service.getViewInfo(
Mockito.eq(viewUrn),
Mockito.any(Authentication.class)
)).thenReturn(
viewInfo
);
return service;
}
private static void verifyMockEntityClient(
EntityClient mockClient,
String entityName,
@ -148,4 +251,28 @@ public class AutoCompleteForMultipleResolverTest {
Mockito.any(Authentication.class)
);
}
private static DataHubViewInfo createViewInfo(StringArray entityNames) {
Filter viewFilter = new Filter()
.setOr(new ConjunctiveCriterionArray(
new ConjunctiveCriterion().setAnd(
new CriterionArray(ImmutableList.of(
new Criterion()
.setField("field")
.setValue("test")
.setValues(new StringArray(ImmutableList.of("test")))
))
)));
DataHubViewInfo info = new DataHubViewInfo();
info.setName("test");
info.setType(DataHubViewType.GLOBAL);
info.setCreated(new AuditStamp().setTime(0L).setActor(TEST_USER_URN));
info.setLastModified(new AuditStamp().setTime(0L).setActor(TEST_USER_URN));
info.setDefinition(new DataHubViewDefinition()
.setEntityTypes(entityNames)
.setFilter(viewFilter)
);
return info;
}
}

7
datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/extensions/mentions/MentionsComponent.tsx
View File

@ -7,6 +7,7 @@ import { Positioner, selectionPositioner } from 'remirror/extensions';
import { useGetAutoCompleteMultipleResultsLazyQuery } from '../../../../../../../../../graphql/search.generated';
import { MentionsDropdown } from './MentionsDropdown';
import { useDataHubMentions } from './useDataHubMentions';
import { useUserContext } from '../../../../../../../../context/useUserContext';
const Container = styled.div`
position: relative;
@ -19,15 +20,17 @@ const StyledEmpty = styled(Empty)`
`;
export const MentionsComponent = () => {
const userContext = useUserContext();
const [getAutoComplete, { data: autocompleteData, loading }] = useGetAutoCompleteMultipleResultsLazyQuery();
const { active, range, filter: query } = useDataHubMentions({});
const [suggestions, setSuggestions] = useState<any[]>([]);
const viewUrn = userContext.localState?.selectedViewUrn;
useEffect(() => {
if (query) {
getAutoComplete({ variables: { input: { query } } });
getAutoComplete({ variables: { input: { query, viewUrn } } });
}
}, [getAutoComplete, query]);
}, [getAutoComplete, query, viewUrn]);
useDebounce(() => setSuggestions(autocompleteData?.autoCompleteForMultiple?.suggestions || []), 250, [
autocompleteData,
]);

6
datahub-web-react/src/app/home/HomePageHeader.tsx
View File

@ -142,11 +142,13 @@ export const HomePageHeader = () => {
const history = useHistory();
const entityRegistry = useEntityRegistry();
const [getAutoCompleteResultsForMultiple, { data: suggestionsData }] = useGetAutoCompleteMultipleResultsLazyQuery();
const user = useUserContext()?.user;
const userContext = useUserContext();
const themeConfig = useTheme();
const appConfig = useAppConfig();
const [newSuggestionData, setNewSuggestionData] = useState<GetAutoCompleteMultipleResultsQuery | undefined>();
const { selectedQuickFilter } = useQuickFiltersContext();
const { user } = userContext;
const viewUrn = userContext.localState?.selectedViewUrn;
useEffect(() => {
if (suggestionsData !== undefined) {
@ -180,6 +182,7 @@ export const HomePageHeader = () => {
input: {
query,
limit: 10,
viewUrn,
...getAutoCompleteInputFromQuickFilter(selectedQuickFilter),
},
},
@ -207,6 +210,7 @@ export const HomePageHeader = () => {
count: 6,
filters: [],
orFilters: [],
viewUrn,
},
},
});

8
datahub-web-react/src/app/search/SearchablePage.tsx
View File

@ -59,8 +59,10 @@ export const SearchablePage = ({ onSearch, onAutoComplete, children }: Props) =>
const { selectedQuickFilter } = useQuickFiltersContext();
const [getAutoCompleteResults, { data: suggestionsData }] = useGetAutoCompleteMultipleResultsLazyQuery();
const user = useUserContext()?.user;
const userContext = useUserContext();
const [newSuggestionData, setNewSuggestionData] = useState<GetAutoCompleteMultipleResultsQuery | undefined>();
const { user } = userContext;
const viewUrn = userContext.localState?.selectedViewUrn;
useEffect(() => {
if (suggestionsData !== undefined) {
@ -97,6 +99,7 @@ export const SearchablePage = ({ onSearch, onAutoComplete, children }: Props) =>
variables: {
input: {
query,
viewUrn,
...getAutoCompleteInputFromQuickFilter(selectedQuickFilter),
},
},
@ -111,11 +114,12 @@ export const SearchablePage = ({ onSearch, onAutoComplete, children }: Props) =>
variables: {
input: {
query: currentQuery,
viewUrn,
},
},
});
}
}, [currentQuery, getAutoCompleteResults]);
}, [currentQuery, getAutoCompleteResults, viewUrn]);
return (
<>

2
datahub-web-react/src/app/search/autoComplete/quickFilters/QuickFilters.tsx
View File

@ -16,7 +16,7 @@ export default function QuickFilters() {
return (
<QuickFiltersWrapper>
{quickFilters?.map((quickFilter) => (
<QuickFilter quickFilter={quickFilter} />
<QuickFilter key={quickFilter.value} quickFilter={quickFilter} />
))}
</QuickFiltersWrapper>
);

15
datahub-web-react/src/providers/QuickFiltersProvider.tsx
View File

@ -2,9 +2,13 @@ import React, { useEffect, useState } from 'react';
import { QuickFiltersContext } from './QuickFiltersContext';
import { QuickFilter } from '../types.generated';
import { useGetQuickFiltersQuery } from '../graphql/quickFilters.generated';
import { useUserContext } from '../app/context/useUserContext';
export default function QuickFiltersProvider({ children }: { children: React.ReactNode }) {
const { data } = useGetQuickFiltersQuery({ variables: { input: {} } });
const userContext = useUserContext();
const viewUrn = userContext.localState?.selectedViewUrn;
const { data, refetch } = useGetQuickFiltersQuery({ variables: { input: { viewUrn } } });
const [quickFilters, setQuickFilters] = useState<QuickFilter[] | null>(null);
const [selectedQuickFilter, setSelectedQuickFilter] = useState<QuickFilter | null>(null);
@ -14,6 +18,15 @@ export default function QuickFiltersProvider({ children }: { children: React.Rea
}
}, [data, quickFilters]);
// refetch and update quick filters whenever viewUrn changes
useEffect(() => {
refetch({ input: { viewUrn } }).then((result) => {
if (result.data.getQuickFilters?.quickFilters) {
setQuickFilters(result.data.getQuickFilters.quickFilters as QuickFilter[]);
}
});
}, [viewUrn, refetch]);
return (
<QuickFiltersContext.Provider
value={{ quickFilters, setQuickFilters, selectedQuickFilter, setSelectedQuickFilter }}