yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 18:07:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(graphql): add graphql types for business glossary (#2485)

Browse Source 
Co-authored-by: shubham.garg <shubham.garg@thoughtworks.com>
This commit is contained in:
shubham garg 2021-05-12 05:59:00 +05:30 committed by GitHub
parent eb5a28e2e0
commit 95782b1acf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 407 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsClientFactory.java
View File

@ -13,7 +13,7 @@ import com.linkedin.tag.client.Tags;
import com.linkedin.util.Configuration;
import com.linkedin.datajob.client.DataFlows;
import com.linkedin.datajob.client.DataJobs;
import com.linkedin.glossary.client.GlossaryTerms;
/**
* Provides access to clients for use in fetching data from downstream GMS services.
@ -47,6 +47,7 @@ public class GmsClientFactory {
private static Tags _tags;
private static DataFlows _dataFlows;
private static DataJobs _dataJobs;
private static GlossaryTerms _glossaryTerms;
private GmsClientFactory() { }
@ -160,4 +161,15 @@ public class GmsClientFactory {
}
return _tags;
}
public static GlossaryTerms getGlossaryTermsClient() {
if (_glossaryTerms == null) {
synchronized (GmsClientFactory.class) {
if (_glossaryTerms == null) {
_glossaryTerms = new GlossaryTerms(REST_CLIENT);
}
}
}
return _glossaryTerms;
}
}

10
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
View File

@ -40,7 +40,7 @@ import com.linkedin.datahub.graphql.types.tag.TagType;
import com.linkedin.datahub.graphql.types.mlmodel.MLModelType;
import com.linkedin.datahub.graphql.types.dataflow.DataFlowType;
import com.linkedin.datahub.graphql.types.datajob.DataJobType;
import com.linkedin.datahub.graphql.types.glossary.GlossaryTermType;
import graphql.schema.idl.RuntimeWiring;
import org.apache.commons.io.IOUtils;
@ -84,6 +84,7 @@ public class GmsGraphQLEngine {
public static final MLModelType ML_MODEL_TYPE = new MLModelType(GmsClientFactory.getMLModelsClient());
public static final DataFlowType DATA_FLOW_TYPE = new DataFlowType(GmsClientFactory.getDataFlowsClient());
public static final DataJobType DATA_JOB_TYPE = new DataJobType(GmsClientFactory.getDataJobsClient());
public static final GlossaryTermType GLOSSARY_TERM_TYPE = new GlossaryTermType(GmsClientFactory.getGlossaryTermsClient());
/**
* Configures the graph objects that can be fetched primary key.
@ -97,7 +98,8 @@ public class GmsGraphQLEngine {
TAG_TYPE,
ML_MODEL_TYPE,
DATA_FLOW_TYPE,
DATA_JOB_TYPE
DATA_JOB_TYPE,
GLOSSARY_TERM_TYPE
);
/**
@ -225,6 +227,10 @@ public class GmsGraphQLEngine {
new LoadableTypeResolver<>(
DATA_JOB_TYPE,
(env) -> env.getArgument(URN_FIELD_NAME))))
.dataFetcher("glossaryTerm", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
GLOSSARY_TERM_TYPE,
(env) -> env.getArgument(URN_FIELD_NAME))))
);
}

138
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java Normal file
View File

@ -0,0 +1,138 @@
package com.linkedin.datahub.graphql.types.glossary;
import com.google.common.collect.ImmutableSet;
import com.linkedin.common.urn.GlossaryTermUrn;
import com.linkedin.data.template.StringArray;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.types.BrowsableEntityType;
import com.linkedin.datahub.graphql.types.SearchableEntityType;
import com.linkedin.datahub.graphql.generated.AutoCompleteResults;
import com.linkedin.datahub.graphql.generated.BrowsePath;
import com.linkedin.datahub.graphql.generated.BrowseResults;
import com.linkedin.datahub.graphql.generated.GlossaryTerm;
import com.linkedin.datahub.graphql.generated.FacetFilterInput;
import com.linkedin.datahub.graphql.generated.SearchResults;
import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper;
import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper;
import com.linkedin.datahub.graphql.types.mappers.BrowseResultMetadataMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermMapper;
import com.linkedin.datahub.graphql.types.mappers.SearchResultsMapper;
import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
import com.linkedin.glossary.client.GlossaryTerms;
import com.linkedin.metadata.query.AutoCompleteResult;
import com.linkedin.metadata.query.BrowseResult;
import com.linkedin.restli.common.CollectionResponse;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import static com.linkedin.datahub.graphql.Constants.BROWSE_PATH_DELIMITER;
public class GlossaryTermType implements SearchableEntityType<GlossaryTerm>, BrowsableEntityType<GlossaryTerm> {
private static final Set<String> FACET_FIELDS = ImmutableSet.of("");
private static final String DEFAULT_AUTO_COMPLETE_FIELD = "definition";
private final GlossaryTerms _glossaryTermsClient;
public GlossaryTermType(final GlossaryTerms glossaryTermsClient) {
_glossaryTermsClient = glossaryTermsClient;
}
@Override
public Class<GlossaryTerm> objectClass() {
return GlossaryTerm.class;
}
@Override
public EntityType type() {
return EntityType.GLOSSARY_TERM;
}
@Override
public List<GlossaryTerm> batchLoad(final List<String> urns, final QueryContext context) {
final List<GlossaryTermUrn> glossaryTermUrns = urns.stream()
.map(GlossaryTermUtils::getGlossaryTermUrn)
.collect(Collectors.toList());
try {
final Map<GlossaryTermUrn, com.linkedin.glossary.GlossaryTerm> glossaryTermMap = _glossaryTermsClient.batchGet(glossaryTermUrns
.stream()
.filter(Objects::nonNull)
.collect(Collectors.toSet()));
final List<com.linkedin.glossary.GlossaryTerm> gmsResults = new ArrayList<>();
for (GlossaryTermUrn urn : glossaryTermUrns) {
gmsResults.add(glossaryTermMap.getOrDefault(urn, null));
}
return gmsResults.stream()
.map(gmsGlossaryTerm -> gmsGlossaryTerm == null ? null : GlossaryTermMapper.map(gmsGlossaryTerm))
.collect(Collectors.toList());
} catch (Exception e) {
throw new RuntimeException("Failed to batch load GlossaryTerms", e);
}
}
@Override
public SearchResults search(@Nonnull String query,
@Nullable List<FacetFilterInput> filters,
int start,
int count,
@Nonnull final QueryContext context) throws Exception {
final Map<String, String> facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS);
final CollectionResponse<com.linkedin.glossary.GlossaryTerm> searchResult = _glossaryTermsClient.search(query, facetFilters, start, count);
return SearchResultsMapper.map(searchResult, GlossaryTermMapper::map);
}
@Override
public AutoCompleteResults autoComplete(@Nonnull String query,
@Nullable String field,
@Nullable List<FacetFilterInput> filters,
int limit,
@Nonnull final QueryContext context) throws Exception {
final Map<String, String> facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS);
field = field != null ? field : DEFAULT_AUTO_COMPLETE_FIELD;
final AutoCompleteResult result = _glossaryTermsClient.autoComplete(query, field, facetFilters, limit);
return AutoCompleteResultsMapper.map(result);
}
@Override
public BrowseResults browse(@Nonnull List<String> path,
@Nullable List<FacetFilterInput> filters,
int start,
int count,
@Nonnull final QueryContext context) throws Exception {
final Map<String, String> facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS);
final String pathStr = path.size() > 0 ? BROWSE_PATH_DELIMITER + String.join(BROWSE_PATH_DELIMITER, path) : "";
final BrowseResult result = _glossaryTermsClient.browse(
pathStr,
facetFilters,
start,
count);
final List<String> urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList());
final List<GlossaryTerm> glossaryTerms = batchLoad(urns, context);
final BrowseResults browseResults = new BrowseResults();
browseResults.setStart(result.getFrom());
browseResults.setCount(result.getPageSize());
browseResults.setTotal(result.getNumEntities());
browseResults.setMetadata(BrowseResultMetadataMapper.map(result.getMetadata()));
browseResults.setEntities(glossaryTerms.stream()
.map(glossaryTerm -> (com.linkedin.datahub.graphql.generated.Entity) glossaryTerm)
.collect(Collectors.toList()));
return browseResults;
}
@Override
public List<BrowsePath> browsePaths(@Nonnull String urn, @Nonnull final QueryContext context) throws Exception {
final StringArray result = _glossaryTermsClient.getBrowsePaths(GlossaryTermUtils.getGlossaryTermUrn(urn));
return BrowsePathsMapper.map(result);
}
}

18
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermUtils.java Normal file
View File

@ -0,0 +1,18 @@
package com.linkedin.datahub.graphql.types.glossary;
import com.linkedin.common.urn.GlossaryTermUrn;
import java.net.URISyntaxException;
public class GlossaryTermUtils {
private GlossaryTermUtils() { }
static GlossaryTermUrn getGlossaryTermUrn(String urnStr) {
try {
return GlossaryTermUrn.createFromString(urnStr);
} catch (URISyntaxException e) {
throw new RuntimeException(String.format("Failed to retrieve glossary with urn %s, invalid urn", urnStr));
}
}
}

38
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermInfoMapper.java Normal file
View File

@ -0,0 +1,38 @@
package com.linkedin.datahub.graphql.types.glossary.mappers;
import javax.annotation.Nonnull;
import com.linkedin.datahub.graphql.generated.GlossaryTermInfo;
import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
/**
* Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
*
* To be replaced by auto-generated mappers implementations
*/
public class GlossaryTermInfoMapper implements ModelMapper<com.linkedin.glossary.GlossaryTermInfo, GlossaryTermInfo> {
public static final GlossaryTermInfoMapper INSTANCE = new GlossaryTermInfoMapper();
public static GlossaryTermInfo map(@Nonnull final com.linkedin.glossary.GlossaryTermInfo glossaryTermInfo) {
return INSTANCE.apply(glossaryTermInfo);
}
@Override
public GlossaryTermInfo apply(@Nonnull final com.linkedin.glossary.GlossaryTermInfo glossaryTermInfo) {
com.linkedin.datahub.graphql.generated.GlossaryTermInfo glossaryTermInfoResult = new com.linkedin.datahub.graphql.generated.GlossaryTermInfo();
glossaryTermInfoResult.setDefinition(glossaryTermInfo.getDefinition());
glossaryTermInfoResult.setTermSource(glossaryTermInfo.getTermSource());
if (glossaryTermInfo.hasSourceRef()) {
glossaryTermInfoResult.setSourceRef(glossaryTermInfo.getSourceRef());
}
if (glossaryTermInfo.hasSourceUrl()) {
glossaryTermInfoResult.setSourceUrl(glossaryTermInfo.getSourceUrl().toString());
}
if (glossaryTermInfo.hasCustomProperties()) {
glossaryTermInfoResult.setCustomProperties(StringMapMapper.map(glossaryTermInfo.getCustomProperties()));
}
return glossaryTermInfoResult;
}
}

31
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java Normal file
View File

@ -0,0 +1,31 @@
package com.linkedin.datahub.graphql.types.glossary.mappers;
import javax.annotation.Nonnull;
import com.linkedin.datahub.graphql.generated.GlossaryTerm;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
/**
* Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
*
* To be replaced by auto-generated mappers implementations
*/
public class GlossaryTermMapper implements ModelMapper<com.linkedin.glossary.GlossaryTerm, GlossaryTerm> {
public static final GlossaryTermMapper INSTANCE = new GlossaryTermMapper();
public static GlossaryTerm map(@Nonnull final com.linkedin.glossary.GlossaryTerm glossaryTerm) {
return INSTANCE.apply(glossaryTerm);
}
@Override
public GlossaryTerm apply(@Nonnull final com.linkedin.glossary.GlossaryTerm glossaryTerm) {
com.linkedin.datahub.graphql.generated.GlossaryTerm result = new com.linkedin.datahub.graphql.generated.GlossaryTerm();
result.setUrn(glossaryTerm.getUrn().toString());
result.setType(EntityType.GLOSSARY_TERM);
result.setName(glossaryTerm.getUrn().getNameEntity());
result.setGlossaryTermInfo(GlossaryTermInfoMapper.map(glossaryTerm.getGlossaryTermInfo()));
return result;
}
}

58
datahub-graphql-core/src/main/resources/gms.graphql
View File

@ -83,6 +83,10 @@ enum EntityType {
The DATA_JOB Entity
"""
DATA_JOB
"""
The GlossaryTerm Entity
"""
GLOSSARY_TERM
}
type Query {
@ -145,6 +149,11 @@ type Query {
Retrieve the browse path(s) corresponding to an entity
"""
browsePaths(input: BrowsePathsInput!): [BrowsePath!]
"""
Fetch a GlossaryTerm by primary key
"""
glossaryTerm(urn: String!): GlossaryTerm
}
type Mutation {
@ -270,6 +279,55 @@ type Dataset implements EntityWithRelationships & Entity {
globalTags: GlobalTags
}
type GlossaryTerm implements Entity {
"""
Urn of the data platform
"""
urn: String!
"""
GMS Entity Type
"""
type: EntityType!
"""
Name of the data platform
"""
name: String!
"""
Details of the Glossary Term
"""
glossaryTermInfo: GlossaryTermInfo!
}
type GlossaryTermInfo {
"""
Name of the glossary term
"""
definition: String!
"""
Term Source of the glossary term
"""
termSource: String!
"""
Source Ref of the glossary term
"""
sourceRef: String
"""
Source Url of the glossary term
"""
sourceUrl: String
"""
Properties of the glossary term
"""
customProperties: [StringMapEntry!]
}
type DataPlatform implements Entity {
"""
Urn of the data platform

13
datahub-web-react/src/graphql/search.graphql
View File

@ -139,6 +139,19 @@ query getSearchResults($input: SearchInput!) {
...globalTagsFields
}
}
... on GlossaryTerm {
name
glossaryTermInfo {
definition
termSource
sourceRef
sourceUrl
customProperties {
key
value
}
}
}
}
matchedFields {
name

7
gms/api/src/main/idl/com.linkedin.glossary.glossaryTerms.restspec.json
View File

@ -109,6 +109,13 @@
"type" : "int"
} ],
"returns" : "com.linkedin.metadata.query.BrowseResult"
}, {
"name" : "getBrowsePaths",
"parameters" : [ {
"name" : "urn",
"type" : "com.linkedin.common.Urn"
} ],
"returns" : "{ \"type\" : \"array\", \"items\" : \"string\" }"
}, {
"name" : "getSnapshot",
"parameters" : [ {

7
gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json
View File

@ -645,6 +645,13 @@
"type" : "int"
} ],
"returns" : "com.linkedin.metadata.query.BrowseResult"
}, {
"name" : "getBrowsePaths",
"parameters" : [ {
"name" : "urn",
"type" : "com.linkedin.common.Urn"
} ],
"returns" : "{ \"type\" : \"array\", \"items\" : \"string\" }"
}, {
"name" : "getSnapshot",
"parameters" : [ {

70
gms/client/src/main/java/com/linkedin/glossary/client/GlossaryTerms.java
View File

@ -9,7 +9,7 @@ import com.linkedin.glossary.GlossaryTermsFindBySearchRequestBuilder;
import com.linkedin.glossary.GlossaryTermsRequestBuilders;
import com.linkedin.metadata.query.Filter;
import com.linkedin.metadata.query.SortCriterion;
import com.linkedin.metadata.restli.BaseSearchableClient;
import com.linkedin.metadata.restli.BaseBrowsableClient;
import com.linkedin.r2.RemoteInvocationException;
import com.linkedin.restli.client.Client;
import com.linkedin.restli.client.GetAllRequest;
@ -17,6 +17,11 @@ import com.linkedin.restli.client.GetRequest;
import com.linkedin.restli.common.CollectionResponse;
import com.linkedin.restli.common.ComplexResourceKey;
import com.linkedin.restli.common.EmptyRecord;
import com.linkedin.metadata.query.AutoCompleteResult;
import com.linkedin.glossary.GlossaryTermsDoAutocompleteRequestBuilder;
import com.linkedin.glossary.GlossaryTermsDoBrowseRequestBuilder;
import com.linkedin.glossary.GlossaryTermsDoGetBrowsePathsRequestBuilder;
import com.linkedin.metadata.query.BrowseResult;
import java.util.List;
import java.util.Map;
@ -26,7 +31,7 @@ import javax.annotation.Nullable;
import static com.linkedin.metadata.dao.utils.QueryUtils.*;
public class GlossaryTerms extends BaseSearchableClient<GlossaryTerm> {
public class GlossaryTerms extends BaseBrowsableClient<GlossaryTerm, GlossaryTermUrn> {
private static final GlossaryTermsRequestBuilders BUSINESS_TERMS_REQUEST_BUILDERS = new GlossaryTermsRequestBuilders();
@ -129,6 +134,67 @@ public class GlossaryTerms extends BaseSearchableClient<GlossaryTerm> {
return search(input, null, null, start, count);
}
/**
* Auto complete glossary terms
*
* @param query search query
* @param field field of the dataset
* @param requestFilters autocomplete filters
* @param limit max number of autocomplete results
* @throws RemoteInvocationException
*/
@Nonnull
public AutoCompleteResult autoComplete(@Nonnull String query, @Nonnull String field,
@Nonnull Map<String, String> requestFilters,
@Nonnull int limit) throws RemoteInvocationException {
GlossaryTermsDoAutocompleteRequestBuilder requestBuilder = BUSINESS_TERMS_REQUEST_BUILDERS
.actionAutocomplete()
.queryParam(query)
.fieldParam(field)
.filterParam(newFilter(requestFilters))
.limitParam(limit);
return _client.sendRequest(requestBuilder.build()).getResponse().getEntity();
}
/**
* Gets browse snapshot of a given path
*
* @param path path being browsed
* @param requestFilters browse filters
* @param start start offset of first dataset
* @param limit max number of datasets
* @throws RemoteInvocationException
*/
@Nonnull
@Override
public BrowseResult browse(@Nonnull String path, @Nullable Map<String, String> requestFilters,
int start, int limit) throws RemoteInvocationException {
GlossaryTermsDoBrowseRequestBuilder requestBuilder = BUSINESS_TERMS_REQUEST_BUILDERS
.actionBrowse()
.pathParam(path)
.startParam(start)
.limitParam(limit);
if (requestFilters != null) {
requestBuilder.filterParam(newFilter(requestFilters));
}
return _client.sendRequest(requestBuilder.build()).getResponse().getEntity();
}
/**
* Gets browse path(s) given glossary term urn
*
* @param urn urn for the entity
* @return list of paths given urn
* @throws RemoteInvocationException
*/
@Nonnull
public StringArray getBrowsePaths(@Nonnull GlossaryTermUrn urn) throws RemoteInvocationException {
GlossaryTermsDoGetBrowsePathsRequestBuilder requestBuilder = BUSINESS_TERMS_REQUEST_BUILDERS
.actionGetBrowsePaths()
.urnParam(urn);
return _client.sendRequest(requestBuilder.build()).getResponse().getEntity();
}
@Nonnull
private GlossaryTermKey toGlossaryTermKey(@Nonnull GlossaryTermUrn urn) {
return new GlossaryTermKey()

8
gms/impl/src/main/java/com/linkedin/metadata/resources/glossary/GlossaryTerms.java
View File

@ -41,6 +41,7 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Named;
import com.linkedin.data.template.StringArray;
import static com.linkedin.metadata.restli.RestliConstants.*;
@ -214,4 +215,11 @@ public final class GlossaryTerms extends BaseBrowsableEntityResource<
return super.browse(path, filter, start, limit);
}
@Action(name = ACTION_GET_BROWSE_PATHS)
@Override
@Nonnull
public Task<StringArray> getBrowsePaths(
@ActionParam(value = "urn", typeref = com.linkedin.common.Urn.class) @Nonnull Urn urn) {
return super.getBrowsePaths(urn);
}
}