yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-26 09:26:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(group ui): Basic group search membership in UI (#3094)

Browse Source
This commit is contained in:
John Joyce 2021-08-16 20:47:18 -07:00 committed by GitHub
parent c9c1ba457e
commit f40bf1ce19
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
70 changed files with 1427 additions and 473 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
datahub-frontend/app/graphql/AnalyticsChart.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public interface AnalyticsChart {

2
datahub-frontend/app/graphql/AnalyticsChartGroup.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public class AnalyticsChartGroup implements java.io.Serializable {

2
datahub-frontend/app/graphql/BarChart.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public class BarChart implements java.io.Serializable, AnalyticsChart {

2
datahub-frontend/app/graphql/BarSegment.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public class BarSegment implements java.io.Serializable {

2
datahub-frontend/app/graphql/DateInterval.java
View File

@ -2,7 +2,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public enum DateInterval {

2
datahub-frontend/app/graphql/DateRange.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public class DateRange implements java.io.Serializable {

2
datahub-frontend/app/graphql/GetAnalyticsChartsQueryResolver.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public interface GetAnalyticsChartsQueryResolver {

2
datahub-frontend/app/graphql/GetHighlightsQueryResolver.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public interface GetHighlightsQueryResolver {

2
datahub-frontend/app/graphql/Highlight.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public class Highlight implements java.io.Serializable {

2
datahub-frontend/app/graphql/NamedBar.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public class NamedBar implements java.io.Serializable {

2
datahub-frontend/app/graphql/NamedLine.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public class NamedLine implements java.io.Serializable {

2
datahub-frontend/app/graphql/NumericDataPoint.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public class NumericDataPoint implements java.io.Serializable {

4
datahub-frontend/app/graphql/QueryResolver.java
View File

@ -3,10 +3,12 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public interface QueryResolver {
boolean isAnalyticsEnabled() throws Exception;
java.util.List<AnalyticsChartGroup> getAnalyticsCharts() throws Exception;
java.util.List<Highlight> getHighlights() throws Exception;

2
datahub-frontend/app/graphql/Row.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public class Row implements java.io.Serializable {

2
datahub-frontend/app/graphql/TableChart.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public class TableChart implements java.io.Serializable, AnalyticsChart {

2
datahub-frontend/app/graphql/TimeSeriesChart.java
View File

@ -3,7 +3,7 @@ package graphql;
@javax.annotation.Generated(
value = "com.kobylynskyi.graphql.codegen.GraphQLCodegen",
date = "2021-05-03T10:56:06-0700"
date = "2021-08-12T10:01:57-0700"
)
public class TimeSeriesChart implements java.io.Serializable, AnalyticsChart {

14
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsClientFactory.java
View File

@ -4,7 +4,7 @@ import com.linkedin.dataplatform.client.DataPlatforms;
import com.linkedin.entity.client.AspectClient;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.lineage.client.Lineages;
import com.linkedin.lineage.client.Relationships;
import com.linkedin.lineage.client.RelationshipClient;
import com.linkedin.metadata.restli.DefaultRestliClientFactory;
import com.linkedin.restli.client.Client;
import com.linkedin.usage.UsageClient;
@ -33,7 +33,7 @@ public class GmsClientFactory {
private static DataPlatforms _dataPlatforms;
private static Lineages _lineages;
private static Relationships _relationships;
private static RelationshipClient _relationshipClient;
private static EntityClient _entities;
private static AspectClient _aspects;
private static UsageClient _usage;
@ -52,15 +52,15 @@ public class GmsClientFactory {
return _lineages;
}
public static Relationships getRelationshipsClient() {
if (_relationships == null) {
public static RelationshipClient getRelationshipsClient() {
if (_relationshipClient == null) {
synchronized (GmsClientFactory.class) {
if (_relationships == null) {
_relationships = new Relationships(REST_CLIENT);
if (_relationshipClient == null) {
_relationshipClient = new RelationshipClient(REST_CLIENT);
}
}
}
return _relationships;
return _relationshipClient;
}
public static EntityClient getEntitiesClient() {

345
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
View File

@ -11,6 +11,7 @@ import com.linkedin.datahub.graphql.generated.DataJobInputOutput;
import com.linkedin.datahub.graphql.generated.Dataset;
import com.linkedin.datahub.graphql.generated.Entity;
import com.linkedin.datahub.graphql.generated.EntityRelationship;
import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy;
import com.linkedin.datahub.graphql.generated.MLModelProperties;
import com.linkedin.datahub.graphql.generated.RelatedDataset;
import com.linkedin.datahub.graphql.generated.SearchResult;
@ -33,6 +34,7 @@ import com.linkedin.datahub.graphql.resolvers.load.AspectResolver;
import com.linkedin.datahub.graphql.resolvers.load.EntityTypeBatchResolver;
import com.linkedin.datahub.graphql.resolvers.load.EntityTypeResolver;
import com.linkedin.datahub.graphql.resolvers.load.LoadableTypeBatchResolver;
import com.linkedin.datahub.graphql.resolvers.load.EntityRelationshipsResultResolver;
import com.linkedin.datahub.graphql.resolvers.load.TimeSeriesAspectResolver;
import com.linkedin.datahub.graphql.resolvers.load.UsageTypeResolver;
import com.linkedin.datahub.graphql.resolvers.mutate.MutableTypeResolver;
@ -78,6 +80,7 @@ import com.linkedin.datahub.graphql.types.glossary.GlossaryTermType;
import com.linkedin.datahub.graphql.types.usage.UsageType;
import graphql.execution.DataFetcherResult;
import graphql.schema.idl.RuntimeWiring;
import java.util.ArrayList;
import org.apache.commons.io.IOUtils;
import org.dataloader.BatchLoaderContextProvider;
import org.dataloader.DataLoader;
@ -222,7 +225,7 @@ public class GmsGraphQLEngine {
public static void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
configureQueryResolvers(builder);
configureMutationResolvers(builder);
configureSearchAndBrowseResolvers(builder);
configureGenericEntityResolvers(builder);
configureDatasetResolvers(builder);
configureCorpUserResolvers(builder);
configureCorpGroupResolvers(builder);
@ -337,7 +340,7 @@ public class GmsGraphQLEngine {
);
}
private static void configureSearchAndBrowseResolvers(final RuntimeWiring.Builder builder) {
private static void configureGenericEntityResolvers(final RuntimeWiring.Builder builder) {
builder
.type("SearchResult", typeWiring -> typeWiring
.dataFetcher("entity", new AuthenticatedResolver<>(
@ -352,6 +355,20 @@ public class GmsGraphQLEngine {
ENTITY_TYPES.stream().collect(Collectors.toList()),
(env) -> ((BrowseResults) env.getSource()).getEntities()))
)
)
.type("EntityRelationshipLegacy", typeWiring -> typeWiring
.dataFetcher("entity", new AuthenticatedResolver<>(
new EntityTypeResolver(
new ArrayList<>(ENTITY_TYPES),
(env) -> ((EntityRelationshipLegacy) env.getSource()).getEntity()))
)
)
.type("EntityRelationship", typeWiring -> typeWiring
.dataFetcher("entity", new AuthenticatedResolver<>(
new EntityTypeResolver(
new ArrayList<>(ENTITY_TYPES),
(env) -> ((EntityRelationship) env.getSource()).getEntity()))
)
);
}
@ -410,13 +427,6 @@ public class GmsGraphQLEngine {
(env) -> ((RelatedDataset) env.getSource()).getDataset().getUrn()))
)
)
.type("EntityRelationship", typeWiring -> typeWiring
.dataFetcher("entity", new AuthenticatedResolver<>(
new EntityTypeResolver(
ENTITY_TYPES.stream().collect(Collectors.toList()),
(env) -> ((EntityRelationship) env.getSource()).getEntity()))
)
)
.type("InstitutionalMemoryMetadata", typeWiring -> typeWiring
.dataFetcher("author", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
@ -430,6 +440,11 @@ public class GmsGraphQLEngine {
* Configures resolvers responsible for resolving the {@link com.linkedin.datahub.graphql.generated.CorpUser} type.
*/
private static void configureCorpUserResolvers(final RuntimeWiring.Builder builder) {
builder.type("CorpUser", typeWiring -> typeWiring
.dataFetcher("relationships", new AuthenticatedResolver<>(
new EntityRelationshipsResultResolver(GmsClientFactory.getRelationshipsClient())
))
);
builder.type("CorpUserInfo", typeWiring -> typeWiring
.dataFetcher("manager", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
@ -443,31 +458,41 @@ public class GmsGraphQLEngine {
* Configures resolvers responsible for resolving the {@link com.linkedin.datahub.graphql.generated.CorpGroup} type.
*/
private static void configureCorpGroupResolvers(final RuntimeWiring.Builder builder) {
builder.type("CorpGroup", typeWiring -> typeWiring
.dataFetcher("relationships", new AuthenticatedResolver<>(
new EntityRelationshipsResultResolver(GmsClientFactory.getRelationshipsClient())
))
);
builder.type("CorpGroupInfo", typeWiring -> typeWiring
.dataFetcher("admins", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
CORP_USER_TYPE,
(env) -> ((CorpGroupInfo) env.getSource()).getAdmins().stream()
.map(CorpUser::getUrn)
.collect(Collectors.toList())))
)
.dataFetcher("members", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
CORP_USER_TYPE,
(env) -> ((CorpGroupInfo) env.getSource()).getMembers().stream()
.map(CorpUser::getUrn)
.collect(Collectors.toList())))
)
.dataFetcher("admins", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
CORP_USER_TYPE,
(env) -> ((CorpGroupInfo) env.getSource()).getAdmins().stream()
.map(CorpUser::getUrn)
.collect(Collectors.toList())))
)
.dataFetcher("members", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
CORP_USER_TYPE,
(env) -> ((CorpGroupInfo) env.getSource()).getMembers().stream()
.map(CorpUser::getUrn)
.collect(Collectors.toList())))
)
);
}
private static void configureTagAssociationResolver(final RuntimeWiring.Builder builder) {
builder.type("Tag", typeWiring -> typeWiring
.dataFetcher("relationships", new AuthenticatedResolver<>(
new EntityRelationshipsResultResolver(GmsClientFactory.getRelationshipsClient())
))
);
builder.type("TagAssociation", typeWiring -> typeWiring
.dataFetcher("tag", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
TAG_TYPE,
(env) -> ((com.linkedin.datahub.graphql.generated.TagAssociation) env.getSource()).getTag().getUrn()))
)
.dataFetcher("tag", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
TAG_TYPE,
(env) -> ((com.linkedin.datahub.graphql.generated.TagAssociation) env.getSource()).getTag().getUrn()))
)
);
}
@ -475,26 +500,29 @@ public class GmsGraphQLEngine {
* Configures resolvers responsible for resolving the {@link com.linkedin.datahub.graphql.generated.Dashboard} type.
*/
private static void configureDashboardResolvers(final RuntimeWiring.Builder builder) {
builder.type("DashboardInfo", typeWiring -> typeWiring
.dataFetcher("charts", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
CHART_TYPE,
(env) -> ((DashboardInfo) env.getSource()).getCharts().stream()
.map(Chart::getUrn)
.collect(Collectors.toList())))
)
);
builder.type("Dashboard", typeWiring -> typeWiring
.dataFetcher("downstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DOWNSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
.dataFetcher("upstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
UPSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
.dataFetcher("relationships", new AuthenticatedResolver<>(
new EntityRelationshipsResultResolver(GmsClientFactory.getRelationshipsClient())
))
.dataFetcher("downstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DOWNSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
.dataFetcher("upstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
UPSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
);
builder.type("DashboardInfo", typeWiring -> typeWiring
.dataFetcher("charts", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
CHART_TYPE,
(env) -> ((DashboardInfo) env.getSource()).getCharts().stream()
.map(Chart::getUrn)
.collect(Collectors.toList())))
)
);
}
@ -503,25 +531,28 @@ public class GmsGraphQLEngine {
*/
private static void configureChartResolvers(final RuntimeWiring.Builder builder) {
builder.type("Chart", typeWiring -> typeWiring
.dataFetcher("downstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DOWNSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
.dataFetcher("upstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
UPSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
.dataFetcher("relationships", new AuthenticatedResolver<>(
new EntityRelationshipsResultResolver(GmsClientFactory.getRelationshipsClient())
))
.dataFetcher("downstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DOWNSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
.dataFetcher("upstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
UPSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
);
builder.type("ChartInfo", typeWiring -> typeWiring
.dataFetcher("inputs", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
DATASET_TYPE,
(env) -> ((ChartInfo) env.getSource()).getInputs().stream()
.map(Dataset::getUrn)
.collect(Collectors.toList())))
)
.dataFetcher("inputs", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
DATASET_TYPE,
(env) -> ((ChartInfo) env.getSource()).getInputs().stream()
.map(Dataset::getUrn)
.collect(Collectors.toList())))
)
);
}
@ -531,17 +562,17 @@ public class GmsGraphQLEngine {
private static void configureTypeResolvers(final RuntimeWiring.Builder builder) {
builder
.type("Entity", typeWiring -> typeWiring
.typeResolver(new EntityInterfaceTypeResolver(LOADABLE_TYPES.stream()
.filter(graphType -> graphType instanceof EntityType)
.map(graphType -> (EntityType<?>) graphType)
.collect(Collectors.toList())
)))
.typeResolver(new EntityInterfaceTypeResolver(LOADABLE_TYPES.stream()
.filter(graphType -> graphType instanceof EntityType)
.map(graphType -> (EntityType<?>) graphType)
.collect(Collectors.toList())
)))
.type("EntityWithRelationships", typeWiring -> typeWiring
.typeResolver(new EntityInterfaceTypeResolver(LOADABLE_TYPES.stream()
.filter(graphType -> graphType instanceof EntityType)
.map(graphType -> (EntityType<?>) graphType)
.collect(Collectors.toList())
)))
.typeResolver(new EntityInterfaceTypeResolver(LOADABLE_TYPES.stream()
.filter(graphType -> graphType instanceof EntityType)
.map(graphType -> (EntityType<?>) graphType)
.collect(Collectors.toList())
)))
.type("OwnerType", typeWiring -> typeWiring
.typeResolver(new EntityInterfaceTypeResolver(OWNER_TYPES.stream()
.filter(graphType -> graphType instanceof EntityType)
@ -573,6 +604,9 @@ public class GmsGraphQLEngine {
private static void configureDataJobResolvers(final RuntimeWiring.Builder builder) {
builder
.type("DataJob", typeWiring -> typeWiring
.dataFetcher("relationships", new AuthenticatedResolver<>(
new EntityRelationshipsResultResolver(GmsClientFactory.getRelationshipsClient())
))
.dataFetcher("dataFlow", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DATA_FLOW_TYPE,
@ -619,91 +653,100 @@ public class GmsGraphQLEngine {
* Configures resolvers responsible for resolving the {@link com.linkedin.datahub.graphql.generated.MLFeatureTable} type.
*/
private static void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builder) {
builder.type("MLModelProperties", typeWiring -> typeWiring
.dataFetcher("groups", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
ML_MODEL_GROUP_TYPE,
(env) -> ((MLModelProperties) env.getSource()).getGroups().stream()
.map(MLModelGroup::getUrn)
.collect(Collectors.toList())))
)
);
builder
.type("MLFeatureTable", typeWiring -> typeWiring
.dataFetcher("platform", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DATA_PLATFORM_TYPE,
(env) -> ((MLFeatureTable) env.getSource()).getPlatform().getUrn()))
)
.dataFetcher("relationships", new AuthenticatedResolver<>(
new EntityRelationshipsResultResolver(GmsClientFactory.getRelationshipsClient())
))
.dataFetcher("platform", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DATA_PLATFORM_TYPE,
(env) -> ((MLFeatureTable) env.getSource()).getPlatform().getUrn()))
)
.type("MLFeatureTableProperties", typeWiring -> typeWiring
.dataFetcher("mlFeatures", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
ML_FEATURE_TYPE,
(env) -> ((MLFeatureTableProperties) env.getSource()).getMlFeatures().stream()
.map(MLFeature::getUrn)
.collect(Collectors.toList())))
)
.dataFetcher("mlPrimaryKeys", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
ML_PRIMARY_KEY_TYPE,
(env) -> ((MLFeatureTableProperties) env.getSource()).getMlPrimaryKeys().stream()
.map(MLPrimaryKey::getUrn)
.collect(Collectors.toList())))
)
)
.type("MLFeatureProperties", typeWiring -> typeWiring
.dataFetcher("sources", new AuthenticatedResolver<>(
)
.type("MLFeatureTableProperties", typeWiring -> typeWiring
.dataFetcher("mlFeatures", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
DATASET_TYPE,
(env) -> ((MLFeatureProperties) env.getSource()).getSources().stream()
.map(Dataset::getUrn)
.collect(Collectors.toList())))
)
ML_FEATURE_TYPE,
(env) -> ((MLFeatureTableProperties) env.getSource()).getMlFeatures().stream()
.map(MLFeature::getUrn)
.collect(Collectors.toList())))
)
.type("MLPrimaryKeyProperties", typeWiring -> typeWiring
.dataFetcher("sources", new AuthenticatedResolver<>(
.dataFetcher("mlPrimaryKeys", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
DATASET_TYPE,
(env) -> ((MLPrimaryKeyProperties) env.getSource()).getSources().stream()
.map(Dataset::getUrn)
.collect(Collectors.toList())))
)
ML_PRIMARY_KEY_TYPE,
(env) -> ((MLFeatureTableProperties) env.getSource()).getMlPrimaryKeys().stream()
.map(MLPrimaryKey::getUrn)
.collect(Collectors.toList())))
)
.type("MLModel", typeWiring -> typeWiring
.dataFetcher("platform", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DATA_PLATFORM_TYPE,
(env) -> ((MLModel) env.getSource()).getPlatform().getUrn()))
)
.dataFetcher("downstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DOWNSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
.dataFetcher("upstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
UPSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
)
.type("MLFeatureProperties", typeWiring -> typeWiring
.dataFetcher("sources", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
DATASET_TYPE,
(env) -> ((MLFeatureProperties) env.getSource()).getSources().stream()
.map(Dataset::getUrn)
.collect(Collectors.toList())))
)
.type("MLModelGroup", typeWiring -> typeWiring
.dataFetcher("platform", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DATA_PLATFORM_TYPE,
(env) -> ((MLModelGroup) env.getSource()).getPlatform().getUrn()))
)
.dataFetcher("downstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DOWNSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
.dataFetcher("upstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
UPSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
);
)
.type("MLPrimaryKeyProperties", typeWiring -> typeWiring
.dataFetcher("sources", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
DATASET_TYPE,
(env) -> ((MLPrimaryKeyProperties) env.getSource()).getSources().stream()
.map(Dataset::getUrn)
.collect(Collectors.toList())))
)
)
.type("MLModel", typeWiring -> typeWiring
.dataFetcher("relationships", new AuthenticatedResolver<>(
new EntityRelationshipsResultResolver(GmsClientFactory.getRelationshipsClient())
))
.dataFetcher("platform", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DATA_PLATFORM_TYPE,
(env) -> ((MLModel) env.getSource()).getPlatform().getUrn()))
)
.dataFetcher("downstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DOWNSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
.dataFetcher("upstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
UPSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
)
.type("MLModelProperties", typeWiring -> typeWiring
.dataFetcher("groups", new AuthenticatedResolver<>(
new LoadableTypeBatchResolver<>(
ML_MODEL_GROUP_TYPE,
(env) -> ((MLModelProperties) env.getSource()).getGroups().stream()
.map(MLModelGroup::getUrn)
.collect(Collectors.toList())))
)
)
.type("MLModelGroup", typeWiring -> typeWiring
.dataFetcher("relationships", new AuthenticatedResolver<>(
new EntityRelationshipsResultResolver(GmsClientFactory.getRelationshipsClient())
))
.dataFetcher("platform", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DATA_PLATFORM_TYPE,
(env) -> ((MLModelGroup) env.getSource()).getPlatform().getUrn()))
)
.dataFetcher("downstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
DOWNSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
.dataFetcher("upstreamLineage", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
UPSTREAM_LINEAGE_TYPE,
(env) -> ((Entity) env.getSource()).getUrn()))
)
);
}

14
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/RelationshipKey.java Normal file
View File

@ -0,0 +1,14 @@
package com.linkedin.datahub.graphql;
import com.linkedin.metadata.query.RelationshipDirection;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class RelationshipKey {
private String urn;
private String relationshipName;
private RelationshipDirection direction; // optional.
}

94
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityRelationshipsResultResolver.java Normal file
View File

@ -0,0 +1,94 @@
package com.linkedin.datahub.graphql.resolvers.load;
import com.linkedin.common.EntityRelationship;
import com.linkedin.common.EntityRelationships;
import com.linkedin.datahub.graphql.generated.Entity;
import com.linkedin.datahub.graphql.generated.EntityRelationshipsResult;
import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper;
import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
import com.linkedin.lineage.client.RelationshipClient;
import com.linkedin.metadata.query.RelationshipDirection;
import com.linkedin.r2.RemoteInvocationException;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.net.URISyntaxException;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
/**
* GraphQL Resolver responsible for fetching relationships between entities in the DataHub graph.
*/
public class EntityRelationshipsResultResolver implements DataFetcher<CompletableFuture<EntityRelationshipsResult>> {
private final RelationshipClient _client;
public EntityRelationshipsResultResolver(final RelationshipClient client) {
_client = client;
}
@Override
public CompletableFuture<EntityRelationshipsResult> get(DataFetchingEnvironment environment) {
final String urn = ((Entity) environment.getSource()).getUrn();
final List<String> relationshipTypes = environment.getArgument("types");
final String relationshipDirection = environment.getArgument("direction");
final Integer start = environment.getArgument("start"); // Optional!
final Integer count = environment.getArgument("count"); // Optional!
final RelationshipDirection resolvedDirection = RelationshipDirection.valueOf(relationshipDirection);
return CompletableFuture.supplyAsync(() -> mapEntityRelationships(
fetchEntityRelationships(
urn,
relationshipTypes,
resolvedDirection,
start,
count
),
resolvedDirection
));
}
private EntityRelationships fetchEntityRelationships(
final String urn,
final List<String> types,
final RelationshipDirection direction,
final Integer start,
final Integer count) {
try {
return _client.getRelationships(urn, direction, types, start, count);
} catch (RemoteInvocationException | URISyntaxException e) {
throw new RuntimeException("Failed to retrieve aspects from GMS", e);
}
}
private EntityRelationshipsResult mapEntityRelationships(
final EntityRelationships entityRelationships,
final RelationshipDirection relationshipDirection
) {
final EntityRelationshipsResult result = new EntityRelationshipsResult();
result.setStart(entityRelationships.getStart());
result.setCount(entityRelationships.getCount());
result.setTotal(entityRelationships.getTotal());
result.setRelationships(entityRelationships.getRelationships().stream().map(entityRelationship -> mapEntityRelationship(
com.linkedin.datahub.graphql.generated.RelationshipDirection.valueOf(relationshipDirection.name()),
entityRelationship)
).collect(Collectors.toList()));
return result;
}
private com.linkedin.datahub.graphql.generated.EntityRelationship mapEntityRelationship(
final com.linkedin.datahub.graphql.generated.RelationshipDirection direction,
final EntityRelationship entityRelationship) {
final com.linkedin.datahub.graphql.generated.EntityRelationship result = new com.linkedin.datahub.graphql.generated.EntityRelationship();
final Entity partialEntity = UrnToEntityMapper.map(entityRelationship.getEntity());
if (partialEntity != null) {
result.setEntity(partialEntity);
}
result.setType(entityRelationship.getType());
result.setDirection(direction);
if (entityRelationship.hasCreated()) {
result.setCreated(AuditStampMapper.map(entityRelationship.getCreated()));
}
return result;
}
}

6
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java
View File

@ -53,11 +53,9 @@ public class TimeSeriesAspectResolver implements DataFetcher<CompletableFuture<L
// Max number of aspects to return.
final Integer maybeLimit = environment.getArgumentOrDefault("limit", null);
List<EnvelopedAspect> aspects;
try {
// Step 1: Get profile aspects.
aspects =
// Step 1: Get aspects.
List<EnvelopedAspect> aspects =
_client.getTimeseriesAspectValues(urn, _entityName, _aspectName, maybeStartTimeMillis, maybeEndTimeMillis,
maybeLimit);

2
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java
View File

@ -63,7 +63,7 @@ public class UrnToEntityMapper implements ModelMapper<com.linkedin.common.urn.Ur
((CorpUser) partialEntity).setUrn(input.toString());
}
if (input.getEntityType().equals("corpGroup")) {
partialEntity = new CorpUser();
partialEntity = new CorpGroup();
((CorpGroup) partialEntity).setUrn(input.toString());
}
if (input.getEntityType().equals("mlFeature")) {

2
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupInfoMapper.java
View File

@ -24,6 +24,8 @@ public class CorpGroupInfoMapper implements ModelMapper<com.linkedin.identity.Co
public CorpGroupInfo apply(@Nonnull final com.linkedin.identity.CorpGroupInfo info) {
final CorpGroupInfo result = new CorpGroupInfo();
result.setEmail(info.getEmail());
result.setDescription(info.getDescription());
result.setDisplayName(info.getDisplayName());
if (info.hasAdmins()) {
result.setAdmins(info.getAdmins().stream().map(urn -> {
final CorpUser corpUser = new CorpUser();

11
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/DataFlowDataJobsRelationshipsType.java
View File

@ -1,10 +1,11 @@
package com.linkedin.datahub.graphql.types.lineage;
import com.google.common.collect.ImmutableList;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.DataFlowDataJobsRelationships;
import com.linkedin.datahub.graphql.types.LoadableType;
import com.linkedin.datahub.graphql.types.relationships.mappers.DataFlowDataJobsRelationshipsMapper;
import com.linkedin.lineage.client.Relationships;
import com.linkedin.lineage.client.RelationshipClient;
import com.linkedin.metadata.query.RelationshipDirection;
import com.linkedin.r2.RemoteInvocationException;
@ -15,11 +16,11 @@ import java.util.stream.Collectors;
public class DataFlowDataJobsRelationshipsType implements LoadableType<DataFlowDataJobsRelationships> {
private final Relationships _relationshipsClient;
private final RelationshipClient _relationshipClientClient;
private final RelationshipDirection _direction = RelationshipDirection.INCOMING;
public DataFlowDataJobsRelationshipsType(final Relationships relationshipsClient) {
_relationshipsClient = relationshipsClient;
public DataFlowDataJobsRelationshipsType(final RelationshipClient relationshipClientClient) {
_relationshipClientClient = relationshipClientClient;
}
@Override
@ -33,7 +34,7 @@ public class DataFlowDataJobsRelationshipsType implements LoadableType<DataFlowD
return keys.stream().map(urn -> {
try {
com.linkedin.common.EntityRelationships relationships =
_relationshipsClient.getRelationships(urn, _direction, "IsPartOf");
_relationshipClientClient.getRelationships(urn, _direction, ImmutableList.of("IsPartOf"), null, null);
return DataFetcherResult.<DataFlowDataJobsRelationships>newResult().data(DataFlowDataJobsRelationshipsMapper.map(relationships)).build();
} catch (RemoteInvocationException | URISyntaxException e) {
throw new RuntimeException(String.format("Failed to batch load DataJobs for DataFlow %s", urn), e);

4
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/relationships/mappers/DataFlowDataJobsRelationshipsMapper.java
View File

@ -19,8 +19,8 @@ public class DataFlowDataJobsRelationshipsMapper implements
@Override
public DataFlowDataJobsRelationships apply(@Nonnull final com.linkedin.common.EntityRelationships input) {
final DataFlowDataJobsRelationships result = new DataFlowDataJobsRelationships();
result.setEntities(input.getEntities().stream().map(
EntityRelationshipMapper::map
result.setEntities(input.getRelationships().stream().map(
EntityRelationshipLegacyMapper::map
).collect(Collectors.toList()));
return result;
}

4
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/relationships/mappers/DownstreamEntityRelationshipsMapper.java
View File

@ -19,8 +19,8 @@ public class DownstreamEntityRelationshipsMapper implements
@Override
public DownstreamEntityRelationships apply(@Nonnull final com.linkedin.common.EntityRelationships input) {
final DownstreamEntityRelationships result = new DownstreamEntityRelationships();
result.setEntities(input.getEntities().stream().map(
EntityRelationshipMapper::map
result.setEntities(input.getRelationships().stream().map(
EntityRelationshipLegacyMapper::map
).collect(Collectors.toList()));
return result;
}

12
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/relationships/mappers/EntityRelationshipMapper.java → datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/relationships/mappers/EntityRelationshipLegacyMapper.java
View File

@ -1,6 +1,6 @@
package com.linkedin.datahub.graphql.types.relationships.mappers;
import com.linkedin.datahub.graphql.generated.EntityRelationship;
import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy;
import com.linkedin.datahub.graphql.generated.EntityWithRelationships;
import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper;
import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
@ -8,17 +8,17 @@ import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
import javax.annotation.Nonnull;
public class EntityRelationshipMapper implements ModelMapper<com.linkedin.common.EntityRelationship, EntityRelationship> {
public class EntityRelationshipLegacyMapper implements ModelMapper<com.linkedin.common.EntityRelationship, EntityRelationshipLegacy> {
public static final EntityRelationshipMapper INSTANCE = new EntityRelationshipMapper();
public static final EntityRelationshipLegacyMapper INSTANCE = new EntityRelationshipLegacyMapper();
public static EntityRelationship map(@Nonnull final com.linkedin.common.EntityRelationship relationship) {
public static EntityRelationshipLegacy map(@Nonnull final com.linkedin.common.EntityRelationship relationship) {
return INSTANCE.apply(relationship);
}
@Override
public EntityRelationship apply(@Nonnull final com.linkedin.common.EntityRelationship relationship) {
final EntityRelationship result = new EntityRelationship();
public EntityRelationshipLegacy apply(@Nonnull final com.linkedin.common.EntityRelationship relationship) {
final EntityRelationshipLegacy result = new EntityRelationshipLegacy();
EntityWithRelationships partialLineageEntity = (EntityWithRelationships) UrnToEntityMapper.map(relationship.getEntity());
if (partialLineageEntity != null) {

4
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/relationships/mappers/UpstreamEntityRelationshipsMapper.java
View File

@ -17,8 +17,8 @@ public class UpstreamEntityRelationshipsMapper implements ModelMapper<com.linked
@Override
public UpstreamEntityRelationships apply(@Nonnull final com.linkedin.common.EntityRelationships input) {
final UpstreamEntityRelationships result = new UpstreamEntityRelationships();
result.setEntities(input.getEntities().stream().map(
EntityRelationshipMapper::map
result.setEntities(input.getRelationships().stream().map(
EntityRelationshipLegacyMapper::map
).collect(Collectors.toList()));
return result;
}

178
datahub-graphql-core/src/main/resources/gms.graphql
View File

@ -19,6 +19,71 @@ interface Entity {
GMS Entity Type
"""
type: EntityType!
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
"""
Result storing relationship information associated with a particular entity.
"""
type EntityRelationshipsResult {
"""
Start offset of the result set
"""
start: Int
"""
Number of results in the returned result set
"""
count: Int
"""
Total number of results in the result set
"""
total: Int
"""
Relationship edges in the result set
"""
relationships: [EntityRelationship!]!
}
"""
The second version of EntityRelationship model, where Entity can be any entity.
TODO - Migrate all entity relationships to this more generic model
"""
type EntityRelationship {
"""
The type of the relationship
"""
type: String!
"""
The direction of the relationship relative to the source entity.
"""
direction: RelationshipDirection!
"""
Entity that is related via lineage
"""
entity: Entity!
"""
An AuditStamp corresponding to the last modification of this relationship
"""
created: AuditStamp
}
"""
Direction between a source and destination node.
"""
enum RelationshipDirection {
INCOMING,
OUTGOING
}
interface Aspect {
@ -49,6 +114,11 @@ interface EntityWithRelationships implements Entity {
Entities downstream of the given entity
"""
downstreamLineage: DownstreamEntityRelationships
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
"""
@ -361,6 +431,11 @@ type Dataset implements EntityWithRelationships & Entity {
If no start / end time are provided, the most recent events will be returned.
"""
datasetProfiles(startTimeMillis: Long, endTimeMillis: Long, limit: Int): [DatasetProfile!]
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type GlossaryTerm implements Entity {
@ -393,6 +468,11 @@ type GlossaryTerm implements Entity {
Details of the Glossary Term
"""
glossaryTermInfo: GlossaryTermInfo!
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type GlossaryTermInfo {
@ -447,6 +527,11 @@ type DataPlatform implements Entity {
Metadata associated with a dataplatform
"""
info: DataPlatformInfo
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type DataPlatformInfo {
@ -831,28 +916,28 @@ type DataFlowEditableProperties {
description: String
}
type EntityRelationship {
"""
An AuditStamp corresponding to the last modification of this relationship
"""
created: AuditStamp
type EntityRelationshipLegacy {
"""
Entity that is related via lineage
"""
entity: EntityWithRelationships
"""
An AuditStamp corresponding to the last modification of this relationship
"""
created: AuditStamp
}
type UpstreamEntityRelationships {
entities: [EntityRelationship]
entities: [EntityRelationshipLegacy]
}
type DownstreamEntityRelationships {
entities: [EntityRelationship]
entities: [EntityRelationshipLegacy]
}
type DataFlowDataJobsRelationships {
entities: [EntityRelationship]
entities: [EntityRelationshipLegacy]
}
type UpstreamLineage {
@ -1047,6 +1132,11 @@ type CorpUser implements Entity {
The structured tags associated with the user
"""
globalTags: GlobalTags
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type CorpUserInfo {
@ -1140,17 +1230,33 @@ type CorpGroup implements Entity {
type: EntityType!
"""
group name e.g. wherehows-dev, ask_metadata
Group name e.g. wherehows-dev, ask_metadata
"""
name: String
name: String!
"""
Information of the corp group
"""
info: CorpGroupInfo
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type CorpGroupInfo {
"""
The name to display when rendering the group
"""
displayName: String
"""
The description provided for the group.
"""
description: String
"""
email of this group
"""
@ -1220,6 +1326,11 @@ type Tag implements Entity {
Ownership metadata of the dataset
"""
ownership: Ownership
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type TagAssociation {
@ -1846,6 +1957,11 @@ type Dashboard implements EntityWithRelationships & Entity {
Editable properties
"""
editableProperties: DashboardEditableProperties
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type DashboardInfo {
@ -1972,6 +2088,11 @@ type Chart implements EntityWithRelationships & Entity {
Editable properties
"""
editableProperties: ChartEditableProperties
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type ChartInfo {
@ -2211,6 +2332,11 @@ type MLModel implements EntityWithRelationships & Entity {
Entities downstream of the given entity
"""
downstreamLineage: DownstreamEntityRelationships
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type MLModelGroup implements EntityWithRelationships & Entity {
@ -2271,6 +2397,11 @@ type MLModelGroup implements EntityWithRelationships & Entity {
Entities downstream of the given entity
"""
downstreamLineage: DownstreamEntityRelationships
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type MLModelGroupProperties {
@ -2338,6 +2469,11 @@ type MLFeature implements Entity {
Deprecation
"""
deprecation: Deprecation
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type MLHyperParam {
@ -2452,6 +2588,11 @@ type MLPrimaryKey implements Entity {
Deprecation
"""
deprecation: Deprecation
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type MLPrimaryKeyProperties {
@ -2516,6 +2657,11 @@ type MLFeatureTable implements Entity {
Deprecation
"""
deprecation: Deprecation
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type MLFeatureTableProperties {
@ -2875,6 +3021,11 @@ type DataFlow implements Entity {
Editable properties
"""
editableProperties: DataFlowEditableProperties
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type DataFlowInfo {
@ -2964,6 +3115,11 @@ type DataJob implements EntityWithRelationships & Entity {
Editable properties
"""
editableProperties: DataJobEditableProperties
"""
Edges extending from this entity.
"""
relationships(types: [String!]!, direction: RelationshipDirection!, start: Int, count: Int): EntityRelationshipsResult
}
type DataJobInfo {

4
datahub-web-react/src/App.tsx
View File

@ -11,7 +11,7 @@ import EntityRegistry from './app/entity/EntityRegistry';
import { DashboardEntity } from './app/entity/dashboard/DashboardEntity';
import { ChartEntity } from './app/entity/chart/ChartEntity';
import { UserEntity } from './app/entity/user/User';
import { UserGroupEntity } from './app/entity/userGroup/UserGroup';
import { GroupEntity } from './app/entity/group/Group';
import { DatasetEntity } from './app/entity/dataset/DatasetEntity';
import { DataFlowEntity } from './app/entity/dataFlow/DataFlowEntity';
import { DataJobEntity } from './app/entity/dataJob/DataJobEntity';
@ -76,7 +76,7 @@ const App: React.VFC = () => {
register.register(new DashboardEntity());
register.register(new ChartEntity());
register.register(new UserEntity());
register.register(new UserGroupEntity());
register.register(new GroupEntity());
register.register(new TagEntity());
register.register(new DataFlowEntity());
register.register(new DataJobEntity());

4
datahub-web-react/src/app/entity/dataFlow/profile/DataFlowDataJobs.tsx
View File

@ -1,7 +1,7 @@
import { List, Typography } from 'antd';
import React from 'react';
import styled from 'styled-components';
import { EntityType, EntityRelationship } from '../../../../types.generated';
import { EntityType, EntityRelationshipLegacy } from '../../../../types.generated';
import { useEntityRegistry } from '../../../useEntityRegistry';
import { PreviewType } from '../../Entity';
@ -24,7 +24,7 @@ const DataJobItem = styled(List.Item)`
`;
export type Props = {
dataJobs?: (EntityRelationship | null)[] | null;
dataJobs?: (EntityRelationshipLegacy | null)[] | null;
};
export default function DataFlowDataJobs({ dataJobs }: Props) {

21
datahub-web-react/src/app/entity/userGroup/UserGroup.tsx → datahub-web-react/src/app/entity/group/Group.tsx
View File

@ -1,14 +1,14 @@
import { UserOutlined } from '@ant-design/icons';
import * as React from 'react';
import { CorpGroup, EntityType, SearchResult } from '../../../types.generated';
import { CorpGroup, CorpUser, EntityType, SearchResult } from '../../../types.generated';
import { Entity, IconStyleType, PreviewType } from '../Entity';
import { Preview } from './preview/Preview';
import UserGroupProfile from './UserGroupProfile';
import GroupProfile from './GroupProfile';
/**
* Definition of the DataHub CorpGroup entity.
*/
export class UserGroupEntity implements Entity<CorpGroup> {
export class GroupEntity implements Entity<CorpGroup> {
type: EntityType = EntityType.CorpGroup;
// TODO: update icons for UserGroup
@ -31,7 +31,7 @@ export class UserGroupEntity implements Entity<CorpGroup> {
);
};
isSearchEnabled = () => false;
isSearchEnabled = () => true;
isBrowseEnabled = () => false;
@ -39,14 +39,19 @@ export class UserGroupEntity implements Entity<CorpGroup> {
getAutoCompleteFieldName = () => 'name';
getPathName: () => string = () => 'userGroup';
getPathName: () => string = () => 'group';
getCollectionName: () => string = () => 'UserGroups';
getCollectionName: () => string = () => 'Groups';
renderProfile: (urn: string) => JSX.Element = (_) => <UserGroupProfile />;
renderProfile: (urn: string) => JSX.Element = (_) => <GroupProfile />;
renderPreview = (_: PreviewType, data: CorpGroup) => (
<Preview urn={data.urn} name={data.name || data.urn || ''} title={data.name || data.urn || ''} />
<Preview
urn={data.urn}
name={data.info?.displayName || data.name || ''}
description={data.info?.description}
members={data?.relationships?.relationships?.map((rel) => rel?.entity as CorpUser)}
/>
);
renderSearch = (result: SearchResult) => {

47
datahub-web-react/src/app/entity/group/GroupHeader.tsx Normal file
View File

@ -0,0 +1,47 @@
import styled from 'styled-components';
import React from 'react';
import { Space, Typography } from 'antd';
import CustomAvatar from '../../shared/avatar/CustomAvatar';
type Props = {
name?: string | null;
email?: string | null;
description?: string | null;
};
const Row = styled.div`
display: inline-flex;
`;
const AvatarWrapper = styled.div`
margin-right: 32px;
`;
export default function GroupHeader({ name, description, email }: Props) {
// TODO: Add Optional Group Image URLs
return (
<>
<Row>
<AvatarWrapper>
<CustomAvatar size={100} photoUrl={undefined} name={name || undefined} />
</AvatarWrapper>
<div>
<Typography.Title level={3} style={{ marginTop: 8 }}>
{name}
</Typography.Title>
<Space split="|" size="middle">
<a href={`mailto:${email}`}>
<Typography.Text strong>{email}</Typography.Text>
</a>
</Space>
</div>
</Row>
<Typography.Title style={{ marginTop: 40 }} level={5}>
Description
</Typography.Title>
<Space>
<Typography.Paragraph>{description}</Typography.Paragraph>
</Space>
</>
);
}

80
datahub-web-react/src/app/entity/group/GroupMembers.tsx Normal file
View File

@ -0,0 +1,80 @@
import { List, Pagination, Row, Space, Typography } from 'antd';
import React, { useState } from 'react';
import styled from 'styled-components';
import { useGetGroupMembersLazyQuery } from '../../../graphql/group.generated';
import { CorpUser, EntityRelationshipsResult, EntityType } from '../../../types.generated';
import { useEntityRegistry } from '../../useEntityRegistry';
import { PreviewType } from '../Entity';
type Props = {
urn: string;
initialRelationships?: EntityRelationshipsResult | null;
pageSize: number;
};
const MemberList = styled(List)`
&&& {
width: 100%;
border-color: ${(props) => props.theme.styles['border-color-base']};
margin-top: 12px;
margin-bottom: 28px;
padding: 24px 32px;
box-shadow: ${(props) => props.theme.styles['box-shadow']};
}
& li {
padding-top: 28px;
padding-bottom: 28px;
}
& li:not(:last-child) {
border-bottom: 1.5px solid #ededed;
}
`;
const MembersView = styled(Space)`
width: 100%;
margin-bottom: 32px;
padding-top: 28px;
`;
export default function GroupMembers({ urn, initialRelationships, pageSize }: Props) {
const [page, setPage] = useState(1);
const entityRegistry = useEntityRegistry();
const [getMembers, { data: membersData }] = useGetGroupMembersLazyQuery();
const onChangeMembersPage = (newPage: number) => {
setPage(newPage);
const start = (newPage - 1) * pageSize;
getMembers({ variables: { urn, start, count: pageSize } });
};
const relationships = membersData ? membersData.corpGroup?.relationships : initialRelationships;
const total = relationships?.total || 0;
const groupMembers = relationships?.relationships?.map((rel) => rel.entity as CorpUser) || [];
return (
<MembersView direction="vertical" size="middle">
<Typography.Title level={3}>Group Membership</Typography.Title>
<Row justify="center">
<MemberList
dataSource={groupMembers}
split={false}
renderItem={(item, _) => (
<List.Item>
{entityRegistry.renderPreview(EntityType.CorpUser, PreviewType.PREVIEW, item)}
</List.Item>
)}
bordered
/>
<Pagination
current={page}
pageSize={pageSize}
total={total}
showLessItems
onChange={onChangeMembersPage}
showSizeChanger={false}
/>
</Row>
</MembersView>
);
}

105
datahub-web-react/src/app/entity/group/GroupProfile.tsx Normal file
View File

@ -0,0 +1,105 @@
import { Alert } from 'antd';
import React, { useMemo } from 'react';
import GroupHeader from './GroupHeader';
import { useGetGroupQuery } from '../../../graphql/group.generated';
import { useGetAllEntitySearchResults } from '../../../utils/customGraphQL/useGetAllEntitySearchResults';
import useUserParams from '../../shared/entitySearch/routingUtils/useUserParams';
import { EntityProfile } from '../../shared/EntityProfile';
import { EntityRelationshipsResult, EntityType, SearchResult } from '../../../types.generated';
import RelatedEntityResults from '../../shared/entitySearch/RelatedEntityResults';
import { Message } from '../../shared/Message';
import GroupMembers from './GroupMembers';
const messageStyle = { marginTop: '10%' };
export enum TabType {
Members = 'Members',
Ownership = 'Ownership',
}
const ENABLED_TAB_TYPES = [TabType.Members, TabType.Ownership];
const MEMBER_PAGE_SIZE = 20;
/**
* Responsible for reading & writing users.
*/
export default function GroupProfile() {
const { urn } = useUserParams();
const { loading, error, data } = useGetGroupQuery({ variables: { urn, membersCount: MEMBER_PAGE_SIZE } });
const name = data?.corpGroup?.name;
const ownershipResult = useGetAllEntitySearchResults({
query: `owners:${name}`,
});
const contentLoading =
Object.keys(ownershipResult).some((type) => {
return ownershipResult[type].loading;
}) || loading;
const ownershipForDetails = useMemo(() => {
const filteredOwnershipResult: {
[key in EntityType]?: Array<SearchResult>;
} = {};
Object.keys(ownershipResult).forEach((type) => {
const entities = ownershipResult[type].data?.search?.searchResults;
if (entities && entities.length > 0) {
filteredOwnershipResult[type] = ownershipResult[type].data?.search?.searchResults;
}
});
return filteredOwnershipResult;
}, [ownershipResult]);
if (error || (!loading && !error && !data)) {
return <Alert type="error" message={error?.message || 'Group failed to load :('} />;
}
const groupMemberRelationships = data?.corpGroup?.relationships as EntityRelationshipsResult;
const getTabs = () => {
return [
{
name: TabType.Members,
path: TabType.Members.toLocaleLowerCase(),
content: (
<GroupMembers
urn={urn}
initialRelationships={groupMemberRelationships}
pageSize={MEMBER_PAGE_SIZE}
/>
),
},
{
name: TabType.Ownership,
path: TabType.Ownership.toLocaleLowerCase(),
content: <RelatedEntityResults searchResult={ownershipForDetails} />,
},
].filter((tab) => ENABLED_TAB_TYPES.includes(tab.name));
};
const description = data?.corpGroup?.info?.description;
return (
<>
{contentLoading && <Message type="loading" content="Loading..." style={messageStyle} />}
{data && data?.corpGroup && (
<EntityProfile
title=""
tags={null}
header={
<GroupHeader
name={data?.corpGroup?.name}
description={description}
email={data?.corpGroup?.info?.email}
/>
}
tabs={getTabs()}
/>
)}
</>
);
}

62
datahub-web-react/src/app/entity/group/preview/Preview.tsx Normal file
View File

@ -0,0 +1,62 @@
import React from 'react';
import { Avatar, Row, Space, Typography } from 'antd';
import { Link } from 'react-router-dom';
import styled from 'styled-components';
import { CorpUser, EntityType } from '../../../../types.generated';
import { useEntityRegistry } from '../../../useEntityRegistry';
import { CustomAvatar } from '../../../shared/avatar';
const NameText = styled(Typography.Title)`
margin: 0;
color: #0073b1;
`;
const DescriptionText = styled(Typography.Paragraph)`
color: rgba(0, 0, 0, 1);
`;
export const Preview = ({
urn,
name,
description,
members,
}: {
urn: string;
name: string;
description?: string | null;
members?: Array<CorpUser> | null;
}): JSX.Element => {
const entityRegistry = useEntityRegistry();
return (
<Link to={entityRegistry.getEntityUrl(EntityType.CorpGroup, urn)} style={{ width: '100%' }}>
<Row justify="space-between">
<Space direction="vertical" size={4}>
<NameText level={3}>{name}</NameText>
{description?.length === 0 ? (
<DescriptionText type="secondary">No description</DescriptionText>
) : (
<DescriptionText>{description}</DescriptionText>
)}
</Space>
<Avatar.Group maxCount={3} size="default" style={{ marginTop: 12 }}>
{(members || [])?.map((member, key) => (
// eslint-disable-next-line react/no-array-index-key
<div data-testid={`avatar-tag-${member.urn}`} key={`${member.urn}-${key}`}>
<CustomAvatar
name={
member.info?.fullName ||
member.info?.displayName ||
member.info?.firstName ||
member.info?.email
}
url={`/${entityRegistry.getPathName(EntityType.CorpUser)}/${member.urn}`}
photoUrl={member.editableInfo?.pictureLink || undefined}
/>
</div>
))}
</Avatar.Group>
</Row>
</Link>
);
};

2
datahub-web-react/src/app/entity/user/User.tsx
View File

@ -47,7 +47,7 @@ export class UserEntity implements Entity<CorpUser> {
renderPreview = (_: PreviewType, data: CorpUser) => (
<Preview
urn={data.urn}
name={data.info?.displayName || data.urn}
name={data.info?.displayName || data.username}
title={data.info?.title || ''}
photoUrl={data.editableInfo?.pictureLink || undefined}
/>

10
datahub-web-react/src/app/entity/user/UserHeader.tsx
View File

@ -11,7 +11,6 @@ type Props = {
skills?: string[] | null;
teams?: string[] | null;
email?: string | null;
isGroup?: boolean;
};
const Row = styled.div`
@ -31,16 +30,11 @@ const Skills = styled.div`
margin-right: 32px;
`;
export default function UserHeader({ profileSrc, name, title, skills, teams, email, isGroup = false }: Props) {
export default function UserHeader({ profileSrc, name, title, skills, teams, email }: Props) {
return (
<Row>
<AvatarWrapper>
<CustomAvatar
size={100}
photoUrl={profileSrc || undefined}
name={name || undefined}
isGroup={isGroup}
/>
<CustomAvatar size={100} photoUrl={profileSrc || undefined} name={name || undefined} />
</AvatarWrapper>
<div>
<Typography.Title level={3}>{name}</Typography.Title>

52
datahub-web-react/src/app/entity/userGroup/UserGroupProfile.tsx
View File

@ -1,52 +0,0 @@
import { Divider, Alert } from 'antd';
import React from 'react';
import styled from 'styled-components';
import UserHeader from '../user/UserHeader';
import { useGetUserGroupQuery } from '../../../graphql/user.generated';
import { useGetAllEntitySearchResults } from '../../../utils/customGraphQL/useGetAllEntitySearchResults';
import { Message } from '../../shared/Message';
import useUserParams from '../../shared/entitySearch/routingUtils/useUserParams';
const PageContainer = styled.div`
padding: 32px 100px;
`;
const messageStyle = { marginTop: '10%' };
/**
* Responsible for reading & writing users.
*/
export default function UserGroupProfile() {
const { urn } = useUserParams();
const { loading, error, data } = useGetUserGroupQuery({ variables: { urn } });
const name = data?.corpGroup?.name;
const ownershipResult = useGetAllEntitySearchResults({
query: `owners:${name}`,
});
const contentLoading =
Object.keys(ownershipResult).some((type) => {
return ownershipResult[type].loading;
}) || loading;
if (error || (!loading && !error && !data)) {
return <Alert type="error" message={error?.message || 'Entity failed to load'} />;
}
return (
<PageContainer>
{contentLoading && <Message type="loading" content="Loading..." style={messageStyle} />}
<UserHeader
name={data?.corpGroup?.name}
title={data?.corpGroup?.name}
email={data?.corpGroup?.info?.email}
teams={data?.corpGroup?.info?.groups}
isGroup
/>
<Divider />
</PageContainer>
);
}

42
datahub-web-react/src/app/entity/userGroup/preview/Preview.tsx
View File

@ -1,42 +0,0 @@
import React from 'react';
import { Space, Typography } from 'antd';
import { Link } from 'react-router-dom';
import styled from 'styled-components';
import { EntityType } from '../../../../types.generated';
import { useEntityRegistry } from '../../../useEntityRegistry';
import CustomAvatar from '../../../shared/avatar/CustomAvatar';
const NameText = styled(Typography.Title)`
margin: 0;
color: #0073b1;
`;
const TitleText = styled(Typography.Title)`
color: rgba(0, 0, 0, 0.45);
`;
export const Preview = ({
urn,
name,
title,
photoUrl,
}: {
urn: string;
name: string;
title?: string;
photoUrl?: string;
}): JSX.Element => {
const entityRegistry = useEntityRegistry();
return (
<Link to={entityRegistry.getEntityUrl(EntityType.CorpGroup, urn)}>
<Space size={28}>
<CustomAvatar size={60} photoUrl={photoUrl} name={name} isGroup />
<Space direction="vertical" size={4}>
<NameText level={3}>{name}</NameText>
<TitleText>{title}</TitleText>
</Space>
</Space>
</Link>
);
};

65
datahub-web-react/src/graphql/group.graphql Normal file
View File

@ -0,0 +1,65 @@
query getGroup($urn: String!, $membersCount: Int!) {
corpGroup(urn: $urn) {
urn
type
name
info {
displayName
description
email
}
relationships(types: ["IsMemberOfGroup"], direction: INCOMING, start: 0, count: $membersCount) {
start
count
total
relationships {
entity {
... on CorpUser {
urn
username
info {
active
displayName
title
firstName
lastName
fullName
}
editableInfo {
pictureLink
}
}
}
}
}
}
}
query getGroupMembers($urn: String!, $start: Int!, $count: Int!) {
corpGroup(urn: $urn) {
relationships(types: ["IsMemberOfGroup"], direction: INCOMING, start: $start, count: $count) {
start
count
total
relationships {
entity {
... on CorpUser {
urn
username
info {
active
displayName
title
firstName
lastName
fullName
}
editableInfo {
pictureLink
}
}
}
}
}
}
}

30
datahub-web-react/src/graphql/search.graphql
View File

@ -68,6 +68,36 @@ query getSearchResults($input: SearchInput!) {
pictureLink
}
}
... on CorpGroup {
name
info {
displayName
description
}
relationships(types: ["IsMemberOfGroup"], direction: INCOMING) {
relationships {
type
direction
entity {
urn
type
... on CorpUser {
username
info {
active
displayName
title
firstName
lastName
}
editableInfo {
pictureLink
}
}
}
}
}
}
... on Dashboard {
urn
type

48
datahub-web-react/src/graphql/user.graphql
View File

@ -21,51 +21,3 @@ query getUser($urn: String!) {
}
}
}
query getUserGroup($urn: String!) {
corpGroup(urn: $urn) {
urn
type
name
info {
email
admins {
urn
username
info {
active
displayName
title
email
firstName
lastName
fullName
}
editableInfo {
pictureLink
teams
skills
}
}
members {
urn
username
info {
active
displayName
title
email
firstName
lastName
fullName
}
editableInfo {
pictureLink
teams
skills
}
}
groups
}
}
}

14
datahub-web-react/src/utils/sort/topologicalSort.ts
View File

@ -1,12 +1,12 @@
import { EntityRelationship } from '../../types.generated';
import { EntityRelationshipLegacy } from '../../types.generated';
// Sort helper function
function topologicalSortHelper(
node: EntityRelationship,
node: EntityRelationshipLegacy,
explored: Set<string>,
result: Array<EntityRelationship>,
result: Array<EntityRelationshipLegacy>,
urnsArray: Array<string>,
nodes: Array<EntityRelationship>,
nodes: Array<EntityRelationshipLegacy>,
) {
if (!node.entity?.urn) {
return;
@ -29,10 +29,10 @@ function topologicalSortHelper(
}
// Topological Sort function with array of EntityRelationship
export function topologicalSort(input: Array<EntityRelationship | null>) {
export function topologicalSort(input: Array<EntityRelationshipLegacy | null>) {
const explored = new Set<string>();
const result: Array<EntityRelationship> = [];
const nodes: Array<EntityRelationship> = [...input] as Array<EntityRelationship>;
const result: Array<EntityRelationshipLegacy> = [];
const nodes: Array<EntityRelationshipLegacy> = [...input] as Array<EntityRelationshipLegacy>;
const urnsArray: Array<string> = nodes
.filter((node) => !!node.entity?.urn)
.map((node) => node.entity?.urn) as Array<string>;

4
datahub-web-react/src/utils/test-utils/TestPageContainer.tsx
View File

@ -7,7 +7,7 @@ import { DatasetEntity } from '../../app/entity/dataset/DatasetEntity';
import { DataFlowEntity } from '../../app/entity/dataFlow/DataFlowEntity';
import { DataJobEntity } from '../../app/entity/dataJob/DataJobEntity';
import { UserEntity } from '../../app/entity/user/User';
import { UserGroupEntity } from '../../app/entity/userGroup/UserGroup';
import { GroupEntity } from '../../app/entity/group/Group';
import EntityRegistry from '../../app/entity/EntityRegistry';
import { EntityRegistryContext } from '../../entityRegistryContext';
import { TagEntity } from '../../app/entity/tag/Tag';
@ -27,7 +27,7 @@ export function getTestEntityRegistry() {
const entityRegistry = new EntityRegistry();
entityRegistry.register(new DatasetEntity());
entityRegistry.register(new UserEntity());
entityRegistry.register(new UserGroupEntity());
entityRegistry.register(new GroupEntity());
entityRegistry.register(new TagEntity());
entityRegistry.register(new DataFlowEntity());
entityRegistry.register(new DataJobEntity());

10
gms/api/src/main/idl/com.linkedin.lineage.relationships.restspec.json
View File

@ -13,10 +13,18 @@
"type" : "string"
}, {
"name" : "types",
"type" : "string"
"type" : "{ \"type\" : \"array\", \"items\" : \"string\" }"
}, {
"name" : "direction",
"type" : "string"
}, {
"name" : "start",
"type" : "int",
"optional" : true
}, {
"name" : "count",
"type" : "int",
"optional" : true
} ]
}, {
"method" : "delete",

15
gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json
View File

@ -1446,7 +1446,10 @@
"name" : "displayName",
"type" : "string",
"doc" : "The name to use when displaying the group.",
"optional" : true
"optional" : true,
"Searchable" : {
"fieldType" : "TEXT_PARTIAL"
}
}, {
"name" : "email",
"type" : "com.linkedin.common.EmailAddress",
@ -1495,7 +1498,10 @@
"name" : "description",
"type" : "string",
"doc" : "A description of the group.",
"optional" : true
"optional" : true,
"Searchable" : {
"fieldType" : "TEXT_PARTIAL"
}
} ],
"Aspect" : {
"EntityUrns" : [ "com.linkedin.common.CorpGroupUrn" ],
@ -1674,7 +1680,10 @@
"fields" : [ {
"name" : "name",
"type" : "string",
"doc" : "The URL-encoded name of the AD/LDAP group. Serves as a globally unique identifier within DataHub."
"doc" : "The URL-encoded name of the AD/LDAP group. Serves as a globally unique identifier within DataHub.",
"Searchable" : {
"fieldType" : "TEXT_PARTIAL"
}
} ],
"Aspect" : {
"name" : "corpGroupKey"

15
gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json
View File

@ -1772,7 +1772,10 @@
"fields" : [ {
"name" : "name",
"type" : "string",
"doc" : "The URL-encoded name of the AD/LDAP group. Serves as a globally unique identifier within DataHub."
"doc" : "The URL-encoded name of the AD/LDAP group. Serves as a globally unique identifier within DataHub.",
"Searchable" : {
"fieldType" : "TEXT_PARTIAL"
}
} ],
"Aspect" : {
"name" : "corpGroupKey"
@ -1786,7 +1789,10 @@
"name" : "displayName",
"type" : "string",
"doc" : "The name to use when displaying the group.",
"optional" : true
"optional" : true,
"Searchable" : {
"fieldType" : "TEXT_PARTIAL"
}
}, {
"name" : "email",
"type" : "com.linkedin.common.EmailAddress",
@ -1835,7 +1841,10 @@
"name" : "description",
"type" : "string",
"doc" : "A description of the group.",
"optional" : true
"optional" : true,
"Searchable" : {
"fieldType" : "TEXT_PARTIAL"
}
} ],
"Aspect" : {
"EntityUrns" : [ "com.linkedin.common.CorpGroupUrn" ],

15
gms/api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json
View File

@ -1291,7 +1291,10 @@
"name" : "displayName",
"type" : "string",
"doc" : "The name to use when displaying the group.",
"optional" : true
"optional" : true,
"Searchable" : {
"fieldType" : "TEXT_PARTIAL"
}
}, {
"name" : "email",
"type" : "com.linkedin.common.EmailAddress",
@ -1340,7 +1343,10 @@
"name" : "description",
"type" : "string",
"doc" : "A description of the group.",
"optional" : true
"optional" : true,
"Searchable" : {
"fieldType" : "TEXT_PARTIAL"
}
} ],
"Aspect" : {
"EntityUrns" : [ "com.linkedin.common.CorpGroupUrn" ],
@ -1519,7 +1525,10 @@
"fields" : [ {
"name" : "name",
"type" : "string",
"doc" : "The URL-encoded name of the AD/LDAP group. Serves as a globally unique identifier within DataHub."
"doc" : "The URL-encoded name of the AD/LDAP group. Serves as a globally unique identifier within DataHub.",
"Searchable" : {
"fieldType" : "TEXT_PARTIAL"
}
} ],
"Aspect" : {
"name" : "corpGroupKey"

15
gms/api/src/main/snapshot/com.linkedin.identity.corpGroups.snapshot.json
View File

@ -173,7 +173,10 @@
"name" : "displayName",
"type" : "string",
"doc" : "The name to use when displaying the group.",
"optional" : true
"optional" : true,
"Searchable" : {
"fieldType" : "TEXT_PARTIAL"
}
}, {
"name" : "email",
"type" : "com.linkedin.common.EmailAddress",
@ -222,7 +225,10 @@
"name" : "description",
"type" : "string",
"doc" : "A description of the group.",
"optional" : true
"optional" : true,
"Searchable" : {
"fieldType" : "TEXT_PARTIAL"
}
} ],
"Aspect" : {
"EntityUrns" : [ "com.linkedin.common.CorpGroupUrn" ],
@ -245,7 +251,10 @@
"fields" : [ {
"name" : "name",
"type" : "string",
"doc" : "The URL-encoded name of the AD/LDAP group. Serves as a globally unique identifier within DataHub."
"doc" : "The URL-encoded name of the AD/LDAP group. Serves as a globally unique identifier within DataHub.",
"Searchable" : {
"fieldType" : "TEXT_PARTIAL"
}
} ],
"Aspect" : {
"name" : "corpGroupKey"

18
gms/api/src/main/snapshot/com.linkedin.lineage.lineage.snapshot.json
View File

@ -44,6 +44,10 @@
"name" : "entity",
"type" : "Urn",
"doc" : "The downstream dataset the lineage points to"
}, {
"name" : "type",
"type" : "string",
"doc" : "The type of the relationship"
} ]
}, {
"type" : "record",
@ -51,12 +55,24 @@
"namespace" : "com.linkedin.common",
"doc" : "Downstream lineage of a dataset",
"fields" : [ {
"name" : "entities",
"name" : "relationships",
"type" : {
"type" : "array",
"items" : "EntityRelationship"
},
"doc" : "List of related entities"
}, {
"name" : "start",
"type" : "int",
"doc" : "The start of the result set"
}, {
"name" : "count",
"type" : "int",
"doc" : "The start of the result set"
}, {
"name" : "total",
"type" : "int",
"doc" : "Total number of edges found."
} ]
}, "com.linkedin.common.Time", "com.linkedin.common.Urn" ],
"schema" : {

28
gms/api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json
View File

@ -44,6 +44,10 @@
"name" : "entity",
"type" : "Urn",
"doc" : "The downstream dataset the lineage points to"
}, {
"name" : "type",
"type" : "string",
"doc" : "The type of the relationship"
} ]
}, {
"type" : "record",
@ -51,12 +55,24 @@
"namespace" : "com.linkedin.common",
"doc" : "Downstream lineage of a dataset",
"fields" : [ {
"name" : "entities",
"name" : "relationships",
"type" : {
"type" : "array",
"items" : "EntityRelationship"
},
"doc" : "List of related entities"
}, {
"name" : "start",
"type" : "int",
"doc" : "The start of the result set"
}, {
"name" : "count",
"type" : "int",
"doc" : "The start of the result set"
}, {
"name" : "total",
"type" : "int",
"doc" : "Total number of edges found."
} ]
}, "com.linkedin.common.Time", "com.linkedin.common.Urn" ],
"schema" : {
@ -74,10 +90,18 @@
"type" : "string"
}, {
"name" : "types",
"type" : "string"
"type" : "{ \"type\" : \"array\", \"items\" : \"string\" }"
}, {
"name" : "direction",
"type" : "string"
}, {
"name" : "start",
"type" : "int",
"optional" : true
}, {
"name" : "count",
"type" : "int",
"optional" : true
} ]
}, {
"method" : "delete",

5
gms/client/src/main/java/com/linkedin/entity/client/RelationshipClient.java
View File

@ -1,5 +0,0 @@
package com.linkedin.entity.client;
public class RelationshipClient {
// TODO(Gabe): fill this in once the relationship client is merged from Dataflow + Datajob work
}

30
gms/client/src/main/java/com/linkedin/lineage/client/Relationships.java → gms/client/src/main/java/com/linkedin/lineage/client/RelationshipClient.java
View File

@ -2,18 +2,21 @@ package com.linkedin.lineage.client;
import com.linkedin.common.EntityRelationships;
import com.linkedin.common.client.BaseClient;
import com.linkedin.lineage.RelationshipsGetRequestBuilder;
import com.linkedin.lineage.RelationshipsRequestBuilders;
import com.linkedin.metadata.query.RelationshipDirection;
import com.linkedin.r2.RemoteInvocationException;
import com.linkedin.restli.client.Client;
import com.linkedin.restli.client.GetRequest;
import java.util.List;
import javax.annotation.Nonnull;
import java.net.URISyntaxException;
import javax.annotation.Nullable;
public class Relationships extends BaseClient {
public Relationships(@Nonnull Client restliClient) {
public class RelationshipClient extends BaseClient {
public RelationshipClient(@Nonnull Client restliClient) {
super(restliClient);
}
private static final RelationshipsRequestBuilders RELATIONSHIPS_REQUEST_BUILDERS =
@ -23,14 +26,23 @@ public class Relationships extends BaseClient {
* Gets a specific version of downstream {@link EntityRelationships} for the given dataset.
*/
@Nonnull
public EntityRelationships getRelationships(@Nonnull String rawUrn, @Nonnull RelationshipDirection direction, @Nonnull String types)
public EntityRelationships getRelationships(
@Nonnull String rawUrn,
@Nonnull RelationshipDirection direction,
@Nonnull List<String> types,
@Nullable Integer start,
@Nullable Integer count)
throws RemoteInvocationException, URISyntaxException {
final GetRequest<EntityRelationships> request = RELATIONSHIPS_REQUEST_BUILDERS.get()
final RelationshipsGetRequestBuilder requestBuilder = RELATIONSHIPS_REQUEST_BUILDERS.get()
.urnParam(rawUrn)
.directionParam(direction.toString())
.typesParam(types)
.build();
return _client.sendRequest(request).getResponseEntity();
.typesParam(types);
if (start != null) {
requestBuilder.startParam(start);
}
if (count != null) {
requestBuilder.countParam(count);
}
return _client.sendRequest(requestBuilder.build()).getResponseEntity();
}
}

8
gms/impl/src/main/java/com/linkedin/metadata/resources/dataset/DownstreamLineageResource.java
View File

@ -69,7 +69,7 @@ public final class DownstreamLineageResource extends SimpleResourceTemplate<Down
return RestliUtils.toTask(() -> {
final List<DatasetUrn> downstreamUrns = _graphService.findRelatedUrns(
final List<DatasetUrn> downstreamUrns = _graphService.findRelatedEntities(
"dataset",
newFilter("urn", datasetUrn.toString()),
"dataset",
@ -78,11 +78,11 @@ public final class DownstreamLineageResource extends SimpleResourceTemplate<Down
createRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING),
0,
MAX_DOWNSTREAM_CNT
).stream().map(urnStr -> {
).getEntities().stream().map(entity -> {
try {
return DatasetUrn.createFromString(urnStr);
return DatasetUrn.createFromString(entity.getUrn());
} catch (URISyntaxException e) {
throw new RuntimeException(String.format("Failed to convert urn in Neo4j to Urn type %s", urnStr));
throw new RuntimeException(String.format("Failed to convert urn in Neo4j to Urn type %s", entity.getUrn()));
}
}).collect(Collectors.toList());

10
gms/impl/src/main/java/com/linkedin/metadata/resources/lineage/Lineage.java
View File

@ -72,14 +72,14 @@ public final class Lineage extends SimpleResourceTemplate<EntityRelationships> {
private List<Urn> getRelatedEntities(String rawUrn, List<String> relationshipTypes, RelationshipDirection direction) {
return
_graphService.findRelatedUrns("", newFilter("urn", rawUrn),
_graphService.findRelatedEntities("", newFilter("urn", rawUrn),
"", EMPTY_FILTER,
relationshipTypes, createRelationshipFilter(EMPTY_FILTER, direction),
0, MAX_DOWNSTREAM_CNT)
.stream().map(
rawRelatedUrn -> {
.getEntities().stream().map(
entity -> {
try {
return Urn.createFromString(rawRelatedUrn);
return Urn.createFromString(entity.getUrn());
} catch (URISyntaxException e) {
e.printStackTrace();
}
@ -109,7 +109,7 @@ public final class Lineage extends SimpleResourceTemplate<EntityRelationships> {
.collect(Collectors.toList())
);
return new EntityRelationships().setEntities(entityArray);
return new EntityRelationships().setRelationships(entityArray);
});
}
}

72
gms/impl/src/main/java/com/linkedin/metadata/resources/lineage/Relationships.java
View File

@ -5,6 +5,7 @@ import com.linkedin.common.EntityRelationship;
import com.linkedin.common.EntityRelationshipArray;
import com.linkedin.common.EntityRelationships;
import com.linkedin.common.urn.Urn;
import com.linkedin.metadata.graph.RelatedEntitiesResult;
import com.linkedin.metadata.graph.GraphService;
import com.linkedin.metadata.query.CriterionArray;
import com.linkedin.metadata.query.Filter;
@ -13,12 +14,14 @@ import com.linkedin.metadata.restli.RestliUtils;
import com.linkedin.parseq.Task;
import com.linkedin.restli.common.HttpStatus;
import com.linkedin.restli.server.UpdateResponse;
import com.linkedin.restli.server.annotations.Optional;
import com.linkedin.restli.server.annotations.QueryParam;
import com.linkedin.restli.server.annotations.RestLiSimpleResource;
import com.linkedin.restli.server.annotations.RestMethod;
import com.linkedin.restli.server.resources.SimpleResourceTemplate;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Named;
import java.net.URISyntaxException;
@ -47,22 +50,20 @@ public final class Relationships extends SimpleResourceTemplate<EntityRelationsh
super();
}
private List<Urn> getRelatedEntities(String rawUrn, List<String> relationshipTypes, RelationshipDirection direction) {
return
_graphService.findRelatedUrns("", newFilter("urn", rawUrn),
"", EMPTY_FILTER,
relationshipTypes, createRelationshipFilter(EMPTY_FILTER, direction),
0, MAX_DOWNSTREAM_CNT)
.stream().map(
rawRelatedUrn -> {
try {
return Urn.createFromString(rawRelatedUrn);
} catch (URISyntaxException e) {
e.printStackTrace();
}
return null;
}
).collect(Collectors.toList());
private RelatedEntitiesResult getRelatedEntities(
String rawUrn,
List<String> relationshipTypes,
RelationshipDirection direction,
@Nullable Integer start,
@Nullable Integer count) {
start = start == null ? 0 : start;
count = count == null ? MAX_DOWNSTREAM_CNT : count;
return _graphService.findRelatedEntities("", newFilter("urn", rawUrn),
"", EMPTY_FILTER,
relationshipTypes, createRelationshipFilter(EMPTY_FILTER, direction),
start, count);
}
static RelationshipDirection getOppositeDirection(RelationshipDirection direction) {
@ -79,23 +80,42 @@ public final class Relationships extends SimpleResourceTemplate<EntityRelationsh
@RestMethod.Get
public Task<EntityRelationships> get(
@QueryParam("urn") @Nonnull String rawUrn,
@QueryParam("types") @Nonnull String relationshipTypesParam,
@QueryParam("direction") @Nonnull String rawDirection
@QueryParam("types") @Nonnull String[] relationshipTypesParam,
@QueryParam("direction") @Nonnull String rawDirection,
@QueryParam("start") @Optional @Nullable Integer start,
@QueryParam("count") @Optional @Nullable Integer count
) {
RelationshipDirection direction = RelationshipDirection.valueOf(rawDirection);
final List<String> relationshipTypes = Arrays.asList(relationshipTypesParam.split(","));
final List<String> relationshipTypes = Arrays.asList(relationshipTypesParam);
return RestliUtils.toTask(() -> {
final List<Urn> relatedEntities = getRelatedEntities(rawUrn, relationshipTypes, direction);
final RelatedEntitiesResult relatedEntitiesResult = getRelatedEntities(
rawUrn,
relationshipTypes,
direction,
start,
count);
final EntityRelationshipArray entityArray = new EntityRelationshipArray(
relatedEntities.stream().map(
entity -> new EntityRelationship()
.setEntity(entity)
)
.collect(Collectors.toList())
relatedEntitiesResult.getEntities().stream().map(
entity -> {
try {
return new EntityRelationship()
.setEntity(Urn.createFromString(entity.getUrn()))
.setType(entity.getRelationshipType());
} catch (URISyntaxException e) {
throw new RuntimeException(
String.format("Failed to convert urnStr %s found in the Graph to an Urn object", entity.getUrn()));
}
}
).collect(Collectors.toList())
);
return new EntityRelationships().setEntities(entityArray);
return new EntityRelationships()
.setStart(relatedEntitiesResult.getStart())
.setCount(relatedEntitiesResult.getCount())
.setTotal(relatedEntitiesResult.getTotal())
.setRelationships(entityArray);
});
}

3
metadata-ingestion/examples/mce_files/bootstrap_mce.json
View File

@ -93,7 +93,8 @@
"email": "bfoo@linkedin.com",
"admins": ["urn:li:corpuser:jdoe", "urn:li:corpuser:datahub"],
"members": ["urn:li:corpuser:jdoe", "urn:li:corpuser:datahub"],
"groups": ["urn:li:corpGroup:jdoe"]
"groups": ["urn:li:corpGroup:jdoe"],
"description": "This group is full of bfoos!"
}
}
]

92
metadata-ingestion/src/datahub/metadata/schema.avsc
View File

@ -720,6 +720,9 @@
"namespace": "com.linkedin.pegasus2avro.metadata.key",
"fields": [
{
"Searchable": {
"fieldType": "TEXT_PARTIAL"
},
"type": "string",
"name": "name",
"doc": "The URL-encoded name of the AD/LDAP group. Serves as a globally unique identifier within DataHub."
@ -739,6 +742,9 @@
"namespace": "com.linkedin.pegasus2avro.identity",
"fields": [
{
"Searchable": {
"fieldType": "TEXT_PARTIAL"
},
"type": [
"null",
"string"
@ -805,6 +811,9 @@
"doc": "List of groups in this group."
},
{
"Searchable": {
"fieldType": "TEXT_PARTIAL"
},
"type": [
"null",
"string"
@ -2776,6 +2785,13 @@
"doc": "Tags associated with the field"
},
{
"Searchable": {
"/terms/*/urn": {
"boostScore": 0.5,
"fieldName": "fieldGlossaryTerms",
"fieldType": "URN_PARTIAL"
}
},
"type": [
"null",
{
@ -5166,13 +5182,87 @@
"name": "customProperties",
"default": {},
"doc": "A key-value map to capture any other non-standardized properties for the glossary term"
},
{
"type": [
"null",
"string"
],
"name": "rawSchema",
"default": null,
"doc": "Schema definition of the glossary term"
}
],
"doc": "Properties associated with a GlossaryTerm"
},
"com.linkedin.pegasus2avro.common.Ownership",
"com.linkedin.pegasus2avro.common.Status",
"com.linkedin.pegasus2avro.common.BrowsePaths"
"com.linkedin.pegasus2avro.common.BrowsePaths",
{
"type": "record",
"Aspect": {
"name": "glossaryRelatedTerms"
},
"name": "GlossaryRelatedTerms",
"namespace": "com.linkedin.pegasus2avro.glossary",
"fields": [
{
"Relationship": {
"/*": {
"entityTypes": [
"glossaryTerm"
],
"name": "IsA"
}
},
"Searchable": {
"/*": {
"boostScore": 2.0,
"fieldName": "isRelatedTerms",
"fieldType": "URN"
}
},
"type": [
"null",
{
"type": "array",
"items": "string"
}
],
"name": "isRelatedTerms",
"default": null,
"doc": "The relationship Is A with glossary term"
},
{
"Relationship": {
"/*": {
"entityTypes": [
"glossaryTerm"
],
"name": "HasA"
}
},
"Searchable": {
"/*": {
"boostScore": 2.0,
"fieldName": "hasRelatedTerms",
"fieldType": "URN"
}
},
"type": [
"null",
{
"type": "array",
"items": "string"
}
],
"name": "hasRelatedTerms",
"default": null,
"doc": "The relationship Has A with glossary term"
}
],
"doc": "Has A / Is A lineage information about a glossary Term reporting the lineage"
}
]
},
"name": "aspects",

100
metadata-ingestion/src/datahub/metadata/schemas/MetadataChangeEvent.avsc
View File

@ -709,7 +709,10 @@
{
"name": "name",
"type": "string",
"doc": "The URL-encoded name of the AD/LDAP group. Serves as a globally unique identifier within DataHub."
"doc": "The URL-encoded name of the AD/LDAP group. Serves as a globally unique identifier within DataHub.",
"Searchable": {
"fieldType": "TEXT_PARTIAL"
}
}
],
"Aspect": {
@ -729,7 +732,10 @@
"string"
],
"doc": "The name to use when displaying the group.",
"default": null
"default": null,
"Searchable": {
"fieldType": "TEXT_PARTIAL"
}
},
{
"name": "email",
@ -795,7 +801,10 @@
"string"
],
"doc": "A description of the group.",
"default": null
"default": null,
"Searchable": {
"fieldType": "TEXT_PARTIAL"
}
}
],
"Aspect": {
@ -2781,7 +2790,14 @@
}
],
"doc": "Glossary terms associated with the field",
"default": null
"default": null,
"Searchable": {
"/terms/*/urn": {
"boostScore": 0.5,
"fieldName": "fieldGlossaryTerms",
"fieldType": "URN_PARTIAL"
}
}
},
{
"name": "isPartOfKey",
@ -5105,6 +5121,15 @@
},
"doc": "A key-value map to capture any other non-standardized properties for the glossary term",
"default": {}
},
{
"name": "rawSchema",
"type": [
"null",
"string"
],
"doc": "Schema definition of the glossary term",
"default": null
}
],
"Aspect": {
@ -5113,7 +5138,72 @@
},
"com.linkedin.pegasus2avro.common.Ownership",
"com.linkedin.pegasus2avro.common.Status",
"com.linkedin.pegasus2avro.common.BrowsePaths"
"com.linkedin.pegasus2avro.common.BrowsePaths",
{
"type": "record",
"name": "GlossaryRelatedTerms",
"namespace": "com.linkedin.pegasus2avro.glossary",
"doc": "Has A / Is A lineage information about a glossary Term reporting the lineage",
"fields": [
{
"name": "isRelatedTerms",
"type": [
"null",
{
"type": "array",
"items": "string"
}
],
"doc": "The relationship Is A with glossary term",
"default": null,
"Relationship": {
"/*": {
"entityTypes": [
"glossaryTerm"
],
"name": "IsA"
}
},
"Searchable": {
"/*": {
"boostScore": 2.0,
"fieldName": "isRelatedTerms",
"fieldType": "URN"
}
}
},
{
"name": "hasRelatedTerms",
"type": [
"null",
{
"type": "array",
"items": "string"
}
],
"doc": "The relationship Has A with glossary term",
"default": null,
"Relationship": {
"/*": {
"entityTypes": [
"glossaryTerm"
],
"name": "HasA"
}
},
"Searchable": {
"/*": {
"boostScore": 2.0,
"fieldName": "hasRelatedTerms",
"fieldType": "URN"
}
}
}
],
"Aspect": {
"name": "glossaryRelatedTerms"
}
}
]
},
"doc": "The list of metadata aspects associated with the GlossaryTerm. Depending on the use case, this can either be all, or a selection, of supported aspects."

2
metadata-io/src/main/java/com/linkedin/metadata/graph/GraphService.java
View File

@ -12,7 +12,7 @@ public interface GraphService {
void addEdge(final Edge edge);
@Nonnull
List<String> findRelatedUrns(
RelatedEntitiesResult findRelatedEntities(
@Nullable final String sourceType,
@Nonnull final Filter sourceEntityFilter,
@Nullable final String destinationType,

30
metadata-io/src/main/java/com/linkedin/metadata/graph/Neo4jGraphService.java
View File

@ -10,6 +10,7 @@ import com.linkedin.metadata.query.Filter;
import com.linkedin.metadata.query.RelationshipDirection;
import com.linkedin.metadata.query.RelationshipFilter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -77,7 +78,7 @@ public class Neo4jGraphService implements GraphService {
}
@Nonnull
public List<String> findRelatedUrns(
public RelatedEntitiesResult findRelatedEntities(
@Nullable final String sourceType,
@Nonnull final Filter sourceEntityFilter,
@Nullable final String destinationType,
@ -104,27 +105,40 @@ public class Neo4jGraphService implements GraphService {
final RelationshipDirection relationshipDirection = relationshipFilter.getDirection();
String matchTemplate = "MATCH (src%s %s)-[r%s %s]-(dest%s %s) RETURN dest";
String matchTemplate = "MATCH (src%s %s)-[r%s %s]-(dest%s %s)";
if (relationshipDirection == RelationshipDirection.INCOMING) {
matchTemplate = "MATCH (src%s %s)<-[r%s %s]-(dest%s %s) RETURN dest";
matchTemplate = "MATCH (src%s %s)<-[r%s %s]-(dest%s %s)";
} else if (relationshipDirection == RelationshipDirection.OUTGOING) {
matchTemplate = "MATCH (src%s %s)-[r%s %s]->(dest%s %s) RETURN dest";
matchTemplate = "MATCH (src%s %s)-[r%s %s]->(dest%s %s)";
}
final String returnNodes = "RETURN dest, type(r)"; // Return both related entity and the relationship type.
final String returnCount = "RETURN count(*)"; // For getting the total results.
String relationshipTypeFilter = "";
if (relationshipTypes.size() > 0) {
relationshipTypeFilter = ":" + StringUtils.join(relationshipTypes, "|");
}
String statementString =
// Build Statement strings
String baseStatementString =
String.format(matchTemplate, sourceType, srcCriteria, relationshipTypeFilter, edgeCriteria,
destinationType, destCriteria);
statementString += " SKIP $offset LIMIT $count";
final String resultStatementString = String.format("%s %s SKIP $offset LIMIT $count", baseStatementString, returnNodes);
final String countStatementString = String.format("%s %s", baseStatementString, returnCount);
final Statement statement = new Statement(statementString, ImmutableMap.of("offset", offset, "count", count));
// Build Statements
final Statement resultStatement = new Statement(resultStatementString, ImmutableMap.of("offset", offset, "count", count));
final Statement countStatement = new Statement(countStatementString, Collections.emptyMap());
return runQuery(statement).list(record -> record.values().get(0).asNode().get("urn").asString());
// Execute Queries
final List<RelatedEntity> relatedEntities = runQuery(resultStatement).list(record ->
new RelatedEntity(
record.values().get(1).asString(), // Relationship Type
record.values().get(0).asNode().get("urn").asString())); // Urn TODO: Validate this works against Neo4j.
final int totalCount = runQuery(countStatement).single().get(0).asInt();
return new RelatedEntitiesResult(offset, relatedEntities.size(), totalCount, relatedEntities);
}
public void removeNode(@Nonnull final Urn urn) {

14
metadata-io/src/main/java/com/linkedin/metadata/graph/RelatedEntitiesResult.java Normal file
View File

@ -0,0 +1,14 @@
package com.linkedin.metadata.graph;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
@AllArgsConstructor
@Data
public class RelatedEntitiesResult {
int start;
int count;
int total;
List<RelatedEntity> entities;
}

18
metadata-io/src/main/java/com/linkedin/metadata/graph/RelatedEntity.java Normal file
View File

@ -0,0 +1,18 @@
package com.linkedin.metadata.graph;
import lombok.AllArgsConstructor;
import lombok.Data;
@AllArgsConstructor
@Data
public class RelatedEntity {
/**
* How the entity is related, along which edge.
*/
String relationshipType;
/**
* Urn associated with the related entity.
*/
String urn;
}

22
metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java
View File

@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.ImmutableList;
import com.linkedin.common.urn.Urn;
import com.linkedin.metadata.graph.Edge;
import com.linkedin.metadata.graph.RelatedEntity;
import com.linkedin.metadata.graph.RelatedEntitiesResult;
import com.linkedin.metadata.graph.GraphService;
import com.linkedin.metadata.query.Condition;
import com.linkedin.metadata.query.Criterion;
@ -91,7 +93,7 @@ public class ElasticSearchGraphService implements GraphService {
}
@Nonnull
public List<String> findRelatedUrns(
public RelatedEntitiesResult findRelatedEntities(
@Nullable final String sourceType,
@Nonnull final Filter sourceEntityFilter,
@Nullable final String destinationType,
@ -116,13 +118,25 @@ public class ElasticSearchGraphService implements GraphService {
);
if (response == null) {
return ImmutableList.of();
return new RelatedEntitiesResult(offset, 0, 0, ImmutableList.of());
}
return Arrays.stream(response.getHits().getHits())
.map(hit -> ((HashMap<String, String>) hit.getSourceAsMap().getOrDefault(destinationNode, EMPTY_HASH)).getOrDefault("urn", null))
int totalCount = (int) response.getHits().getTotalHits().value;
final List<RelatedEntity> relationships = Arrays.stream(response.getHits().getHits())
.map(hit -> {
final String urnStr = ((HashMap<String, String>) hit.getSourceAsMap().getOrDefault(destinationNode, EMPTY_HASH)).getOrDefault("urn", null);
final String relationshipType = (String) hit.getSourceAsMap().get("relationshipType");
if (urnStr == null || relationshipType == null) {
log.error(String.format(
"Found null urn string or relationship type in Elastic index. urnStr: %s, relationshipType: %s", urnStr, relationshipType));
return null;
}
return new RelatedEntity(relationshipType, urnStr);
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
return new RelatedEntitiesResult(offset, relationships.size(), totalCount, relationships);
}
private Filter createUrnFilter(@Nonnull final Urn urn) {

32
metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java
View File

@ -3,6 +3,7 @@ package com.linkedin.metadata.graph;
import com.linkedin.common.urn.Urn;
import com.linkedin.metadata.query.RelationshipDirection;
import com.linkedin.metadata.query.RelationshipFilter;
import java.util.stream.Collectors;
import org.testng.annotations.Test;
import javax.annotation.Nonnull;
@ -60,7 +61,7 @@ abstract public class GraphServiceTestBase {
relationshipFilter.setDirection(RelationshipDirection.OUTGOING);
relationshipFilter.setCriteria(EMPTY_FILTER.getCriteria());
List<String> relatedUrns = client.findRelatedUrns(
List<String> relatedUrns = client.findRelatedEntities(
"",
newFilter("urn", "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"),
"",
@ -68,7 +69,10 @@ abstract public class GraphServiceTestBase {
edgeTypes,
relationshipFilter,
0,
10);
10).getEntities()
.stream()
.map(RelatedEntity::getUrn)
.collect(Collectors.toList());
assertEquals(relatedUrns.size(), 1);
}
@ -91,7 +95,7 @@ abstract public class GraphServiceTestBase {
relationshipFilter.setDirection(RelationshipDirection.INCOMING);
relationshipFilter.setCriteria(EMPTY_FILTER.getCriteria());
List<String> relatedUrns = client.findRelatedUrns(
List<String> relatedUrns = client.findRelatedEntities(
"",
newFilter("urn", "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"),
"",
@ -99,7 +103,11 @@ abstract public class GraphServiceTestBase {
edgeTypes,
relationshipFilter,
0,
10);
10)
.getEntities()
.stream()
.map(RelatedEntity::getUrn)
.collect(Collectors.toList());
assertEquals(relatedUrns.size(), 1);
}
@ -122,7 +130,7 @@ abstract public class GraphServiceTestBase {
relationshipFilter.setDirection(RelationshipDirection.INCOMING);
relationshipFilter.setCriteria(EMPTY_FILTER.getCriteria());
List<String> relatedUrns = client.findRelatedUrns(
List<String> relatedUrns = client.findRelatedEntities(
"",
newFilter("urn", "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"),
"",
@ -130,7 +138,11 @@ abstract public class GraphServiceTestBase {
edgeTypes,
relationshipFilter,
0,
10);
10)
.getEntities()
.stream()
.map(RelatedEntity::getUrn)
.collect(Collectors.toList());
assertEquals(relatedUrns.size(), 1);
@ -140,7 +152,7 @@ abstract public class GraphServiceTestBase {
relationshipFilter);
syncAfterWrite();
List<String> relatedUrnsPostDelete = client.findRelatedUrns(
List<String> relatedUrnsPostDelete = client.findRelatedEntities(
"",
newFilter("urn", "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"),
"",
@ -148,7 +160,11 @@ abstract public class GraphServiceTestBase {
edgeTypes,
relationshipFilter,
0,
10);
10)
.getEntities()
.stream()
.map(RelatedEntity::getUrn)
.collect(Collectors.toList());
assertEquals(relatedUrnsPostDelete.size(), 0);
}

5
metadata-models/src/main/pegasus/com/linkedin/common/EntityRelationship.pdl
View File

@ -14,4 +14,9 @@ record EntityRelationship {
* The downstream dataset the lineage points to
*/
entity: Urn
/**
* The type of the relationship
*/
type: string
}

18
metadata-models/src/main/pegasus/com/linkedin/common/EntityRelationships.pdl
View File

@ -8,5 +8,21 @@ record EntityRelationships {
/**
* List of related entities
*/
entities: array[EntityRelationship]
relationships: array[EntityRelationship]
/**
* The start of the result set
*/
start: int
/**
* The start of the result set
*/
count: int
/**
* Total number of edges found.
*/
total: int
}

6
metadata-models/src/main/pegasus/com/linkedin/identity/CorpGroupInfo.pdl
View File

@ -16,6 +16,9 @@ record CorpGroupInfo {
/**
* The name to use when displaying the group.
*/
@Searchable = {
"fieldType": "TEXT_PARTIAL"
}
displayName: optional string
/**
@ -59,6 +62,9 @@ record CorpGroupInfo {
/**
* A description of the group.
*/
@Searchable = {
"fieldType": "TEXT_PARTIAL"
}
description: optional string
}

3
metadata-models/src/main/pegasus/com/linkedin/metadata/key/CorpGroupKey.pdl
View File

@ -10,5 +10,8 @@ record CorpGroupKey {
/**
* The URL-encoded name of the AD/LDAP group. Serves as a globally unique identifier within DataHub.
*/
@Searchable = {
"fieldType": "TEXT_PARTIAL"
}
name: string
}