diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 85aa017f6a..c1b08a7c1d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -194,6 +194,9 @@ public class GmsGraphQLEngine { private static void configureMutationResolvers(final RuntimeWiring.Builder builder) { builder.type("Mutation", typeWiring -> typeWiring .dataFetcher("updateDataset", new AuthenticatedResolver<>(new MutableTypeResolver<>(DATASET_TYPE))) + .dataFetcher("updateTag", new AuthenticatedResolver<>(new MutableTypeResolver<>(TAG_TYPE))) + .dataFetcher("updateChart", new AuthenticatedResolver<>(new MutableTypeResolver<>(CHART_TYPE))) + .dataFetcher("updateDashboard", new AuthenticatedResolver<>(new MutableTypeResolver<>(DASHBOARD_TYPE))) ); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java index 17fd3dddfc..319d591435 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java @@ -8,12 +8,15 @@ import com.linkedin.datahub.graphql.generated.AutoCompleteResults; import com.linkedin.datahub.graphql.generated.BrowsePath; import com.linkedin.datahub.graphql.generated.BrowseResults; import com.linkedin.datahub.graphql.generated.Chart; +import com.linkedin.datahub.graphql.generated.ChartUpdateInput; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.BrowsableEntityType; +import com.linkedin.datahub.graphql.types.MutableType; import com.linkedin.datahub.graphql.types.SearchableEntityType; +import com.linkedin.datahub.graphql.types.chart.mappers.ChartUpdateInputMapper; import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; import com.linkedin.datahub.graphql.types.mappers.BrowseResultMetadataMapper; @@ -22,6 +25,7 @@ import com.linkedin.datahub.graphql.types.mappers.SearchResultsMapper; import com.linkedin.metadata.configs.ChartSearchConfig; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; +import com.linkedin.r2.RemoteInvocationException; import com.linkedin.restli.common.CollectionResponse; import javax.annotation.Nonnull; @@ -35,7 +39,7 @@ import java.util.stream.Collectors; import static com.linkedin.datahub.graphql.Constants.BROWSE_PATH_DELIMITER; -public class ChartType implements SearchableEntityType, BrowsableEntityType { +public class ChartType implements SearchableEntityType, BrowsableEntityType, MutableType { private final Charts _chartsClient; private static final ChartSearchConfig CHART_SEARCH_CONFIG = new ChartSearchConfig(); @@ -44,6 +48,11 @@ public class ChartType implements SearchableEntityType, BrowsableEntityTy _chartsClient = chartsClient; } + @Override + public Class inputClass() { + return ChartUpdateInput.class; + } + @Override public EntityType type() { return EntityType.CHART; @@ -139,4 +148,17 @@ public class ChartType implements SearchableEntityType, BrowsableEntityTy throw new RuntimeException(String.format("Failed to retrieve chart with urn %s, invalid urn", urnStr)); } } + + @Override + public Chart update(@Nonnull ChartUpdateInput input, @Nonnull QueryContext context) throws Exception { + final com.linkedin.dashboard.Chart partialChart = ChartUpdateInputMapper.map(input); + + try { + _chartsClient.update(ChartUrn.createFromString(input.getUrn()), partialChart); + } catch (RemoteInvocationException e) { + throw new RuntimeException(String.format("Failed to write entity with urn %s", input.getUrn()), e); + } + + return load(input.getUrn(), context); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartUpdateInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartUpdateInputMapper.java new file mode 100644 index 0000000000..a594c3a0d4 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartUpdateInputMapper.java @@ -0,0 +1,38 @@ +package com.linkedin.datahub.graphql.types.chart.mappers; + +import com.linkedin.common.GlobalTags; +import com.linkedin.common.TagAssociationArray; +import com.linkedin.dashboard.Chart; +import com.linkedin.datahub.graphql.generated.ChartUpdateInput; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.datahub.graphql.types.tag.mappers.TagAssociationUpdateMapper; + +import javax.annotation.Nonnull; +import java.util.stream.Collectors; + +public class ChartUpdateInputMapper implements ModelMapper { + public static final ChartUpdateInputMapper INSTANCE = new ChartUpdateInputMapper(); + + public static Chart map(@Nonnull final ChartUpdateInput chartUpdateInput) { + return INSTANCE.apply(chartUpdateInput); + } + + @Override + public Chart apply(@Nonnull final ChartUpdateInput chartUpdateInput) { + final Chart result = new Chart(); + + if (chartUpdateInput.getGlobalTags() != null) { + final GlobalTags globalTags = new GlobalTags(); + globalTags.setTags( + new TagAssociationArray( + chartUpdateInput.getGlobalTags().getTags().stream().map( + element -> TagAssociationUpdateMapper.map(element) + ).collect(Collectors.toList()) + ) + ); + result.setGlobalTags(globalTags); + } + return result; + } + +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java index 9f8e618ff4..4149663906 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java @@ -8,12 +8,15 @@ import com.linkedin.datahub.graphql.generated.AutoCompleteResults; import com.linkedin.datahub.graphql.generated.BrowsePath; import com.linkedin.datahub.graphql.generated.BrowseResults; import com.linkedin.datahub.graphql.generated.Dashboard; +import com.linkedin.datahub.graphql.generated.DashboardUpdateInput; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.BrowsableEntityType; +import com.linkedin.datahub.graphql.types.MutableType; import com.linkedin.datahub.graphql.types.SearchableEntityType; +import com.linkedin.datahub.graphql.types.dashboard.mappers.DashboardUpdateInputMapper; import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; import com.linkedin.datahub.graphql.types.mappers.BrowseResultMetadataMapper; @@ -22,6 +25,7 @@ import com.linkedin.datahub.graphql.types.mappers.SearchResultsMapper; import com.linkedin.metadata.configs.DashboardSearchConfig; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; +import com.linkedin.r2.RemoteInvocationException; import com.linkedin.restli.common.CollectionResponse; import javax.annotation.Nonnull; @@ -35,7 +39,7 @@ import java.util.stream.Collectors; import static com.linkedin.datahub.graphql.Constants.BROWSE_PATH_DELIMITER; -public class DashboardType implements SearchableEntityType, BrowsableEntityType { +public class DashboardType implements SearchableEntityType, BrowsableEntityType, MutableType { private final Dashboards _dashboardsClient; private static final DashboardSearchConfig DASHBOARDS_SEARCH_CONFIG = new DashboardSearchConfig(); @@ -44,6 +48,11 @@ public class DashboardType implements SearchableEntityType, Browsable _dashboardsClient = dashboardsClient; } + @Override + public Class inputClass() { + return DashboardUpdateInput.class; + } + @Override public EntityType type() { return EntityType.DASHBOARD; @@ -139,4 +148,16 @@ public class DashboardType implements SearchableEntityType, Browsable } } + @Override + public Dashboard update(@Nonnull DashboardUpdateInput input, @Nonnull QueryContext context) throws Exception { + final com.linkedin.dashboard.Dashboard partialDashboard = DashboardUpdateInputMapper.map(input); + + try { + _dashboardsClient.update(DashboardUrn.createFromString(input.getUrn()), partialDashboard); + } catch (RemoteInvocationException e) { + throw new RuntimeException(String.format("Failed to write entity with urn %s", input.getUrn()), e); + } + + return load(input.getUrn(), context); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardUpdateInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardUpdateInputMapper.java new file mode 100644 index 0000000000..b7958a9f0a --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardUpdateInputMapper.java @@ -0,0 +1,38 @@ +package com.linkedin.datahub.graphql.types.dashboard.mappers; + +import com.linkedin.common.GlobalTags; +import com.linkedin.common.TagAssociationArray; +import com.linkedin.dashboard.Dashboard; +import com.linkedin.datahub.graphql.generated.DashboardUpdateInput; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.datahub.graphql.types.tag.mappers.TagAssociationUpdateMapper; + +import javax.annotation.Nonnull; +import java.util.stream.Collectors; + +public class DashboardUpdateInputMapper implements ModelMapper { + public static final DashboardUpdateInputMapper INSTANCE = new DashboardUpdateInputMapper(); + + public static Dashboard map(@Nonnull final DashboardUpdateInput dashboardUpdateInput) { + return INSTANCE.apply(dashboardUpdateInput); + } + + @Override + public Dashboard apply(@Nonnull final DashboardUpdateInput dashboardUpdateInput) { + final Dashboard result = new Dashboard(); + + if (dashboardUpdateInput.getGlobalTags() != null) { + final GlobalTags globalTags = new GlobalTags(); + globalTags.setTags( + new TagAssociationArray( + dashboardUpdateInput.getGlobalTags().getTags().stream().map( + element -> TagAssociationUpdateMapper.map(element) + ).collect(Collectors.toList()) + ) + ); + result.setGlobalTags(globalTags); + } + return result; + } + +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index d8f8be2bf5..3d08885c3d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -167,6 +167,13 @@ public class DatasetType implements SearchableEntityType, BrowsableEnti partialDataset.getDeprecation().setActor(actor, SetMode.IGNORE_NULL); } + if (partialDataset.hasEditableSchemaMetadata()) { + partialDataset.getEditableSchemaMetadata().setLastModified(auditStamp); + if (!partialDataset.getEditableSchemaMetadata().hasCreated()) { + partialDataset.getEditableSchemaMetadata().setCreated(auditStamp); + } + } + partialDataset.setLastModified(auditStamp); try { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index 3dabbb9e56..f6766ec75e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -44,6 +44,9 @@ public class DatasetMapper implements ModelMapper { @@ -36,9 +44,53 @@ public class DatasetUpdateInputMapper implements ModelMapper TagAssociationUpdateMapper.map(element) + ).collect(Collectors.toList()) + ) + ); + result.setGlobalTags(globalTags); + } + + if (datasetUpdateInput.getEditableSchemaMetadata() != null) { + final EditableSchemaMetadata editableSchemaMetadata = new EditableSchemaMetadata(); + editableSchemaMetadata.setEditableSchemaFieldInfo( + new EditableSchemaFieldInfoArray( + datasetUpdateInput.getEditableSchemaMetadata().getEditableSchemaFieldInfo().stream().map( + element -> mapSchemaFieldInfo(element) + ).collect(Collectors.toList()))); + result.setEditableSchemaMetadata(editableSchemaMetadata); + } return result; } + + private EditableSchemaFieldInfo mapSchemaFieldInfo( + final com.linkedin.datahub.graphql.generated.EditableSchemaFieldInfoUpdate schemaFieldInfo + ) { + final EditableSchemaFieldInfo output = new EditableSchemaFieldInfo(); + + if (schemaFieldInfo.getDescription() != null) { + output.setDescription(schemaFieldInfo.getDescription()); + } + output.setFieldPath(schemaFieldInfo.getFieldPath()); + + if (schemaFieldInfo.getGlobalTags() != null) { + final GlobalTags globalTags = new GlobalTags(); + globalTags.setTags(new TagAssociationArray(schemaFieldInfo.getGlobalTags().getTags().stream().map( + element -> TagAssociationUpdateMapper.map(element)).collect(Collectors.toList()))); + output.setGlobalTags(globalTags); + } + + return output; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaFieldInfoMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaFieldInfoMapper.java new file mode 100644 index 0000000000..d4399361de --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaFieldInfoMapper.java @@ -0,0 +1,32 @@ +package com.linkedin.datahub.graphql.types.dataset.mappers; + +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; +import com.linkedin.schema.EditableSchemaFieldInfo; + +import javax.annotation.Nonnull; + + +public class EditableSchemaFieldInfoMapper implements ModelMapper { + + public static final EditableSchemaFieldInfoMapper INSTANCE = new EditableSchemaFieldInfoMapper(); + + public static com.linkedin.datahub.graphql.generated.EditableSchemaFieldInfo map(@Nonnull final EditableSchemaFieldInfo fieldInfo) { + return INSTANCE.apply(fieldInfo); + } + + @Override + public com.linkedin.datahub.graphql.generated.EditableSchemaFieldInfo apply(@Nonnull final EditableSchemaFieldInfo input) { + final com.linkedin.datahub.graphql.generated.EditableSchemaFieldInfo result = new com.linkedin.datahub.graphql.generated.EditableSchemaFieldInfo(); + if (input.hasDescription()) { + result.setDescription((input.getDescription())); + } + if (input.hasFieldPath()) { + result.setFieldPath((input.getFieldPath())); + } + if (input.hasGlobalTags()) { + result.setGlobalTags(GlobalTagsMapper.map(input.getGlobalTags())); + } + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaMetadataMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaMetadataMapper.java new file mode 100644 index 0000000000..84803ed40a --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaMetadataMapper.java @@ -0,0 +1,24 @@ +package com.linkedin.datahub.graphql.types.dataset.mappers; + +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.schema.EditableSchemaMetadata; + +import javax.annotation.Nonnull; +import java.util.stream.Collectors; + +public class EditableSchemaMetadataMapper implements ModelMapper { + + public static final EditableSchemaMetadataMapper INSTANCE = new EditableSchemaMetadataMapper(); + + public static com.linkedin.datahub.graphql.generated.EditableSchemaMetadata map(@Nonnull final EditableSchemaMetadata metadata) { + return INSTANCE.apply(metadata); + } + + @Override + public com.linkedin.datahub.graphql.generated.EditableSchemaMetadata apply(@Nonnull final EditableSchemaMetadata input) { + final com.linkedin.datahub.graphql.generated.EditableSchemaMetadata result = new com.linkedin.datahub.graphql.generated.EditableSchemaMetadata(); + result.setEditableSchemaFieldInfo(input.getEditableSchemaFieldInfo().stream().map(EditableSchemaFieldInfoMapper::map).collect(Collectors.toList())); + return result; + } + +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java index 911ea624cf..273f4776f3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java @@ -1,12 +1,36 @@ package com.linkedin.datahub.graphql.types.tag; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.Owner; +import com.linkedin.common.OwnerArray; +import com.linkedin.common.Ownership; +import com.linkedin.common.OwnershipSource; +import com.linkedin.common.OwnershipSourceType; +import com.linkedin.common.OwnershipType; +import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.TagUrn; +import com.linkedin.data.template.SetMode; import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AutoCompleteResults; import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.SearchResults; import com.linkedin.datahub.graphql.generated.Tag; +import com.linkedin.datahub.graphql.generated.TagUpdate; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.types.MutableType; +import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; +import com.linkedin.datahub.graphql.types.mappers.SearchResultsMapper; import com.linkedin.datahub.graphql.types.tag.mappers.TagMapper; +import com.linkedin.datahub.graphql.types.tag.mappers.TagUpdateMapper; +import com.linkedin.metadata.configs.TagSearchConfig; +import com.linkedin.metadata.query.AutoCompleteResult; +import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.restli.common.CollectionResponse; import com.linkedin.tag.client.Tags; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; @@ -14,9 +38,10 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -public class TagType implements com.linkedin.datahub.graphql.types.EntityType { +public class TagType implements com.linkedin.datahub.graphql.types.SearchableEntityType, MutableType { private static final String DEFAULT_AUTO_COMPLETE_FIELD = "name"; + private static final TagSearchConfig TAG_SEARCH_CONFIG = new TagSearchConfig(); private final Tags _tagClient; @@ -34,6 +59,11 @@ public class TagType implements com.linkedin.datahub.graphql.types.EntityType inputClass() { + return TagUpdate.class; + } + @Override public List batchLoad(final List urns, final QueryContext context) { @@ -59,6 +89,65 @@ public class TagType implements com.linkedin.datahub.graphql.types.EntityType filters, + int start, + int count, + @Nonnull QueryContext context) throws Exception { + final Map facetFilters = ResolverUtils.buildFacetFilters(filters, TAG_SEARCH_CONFIG.getFacetFields()); + final CollectionResponse searchResult = _tagClient.search(query, null, facetFilters, null, start, count); + return SearchResultsMapper.map(searchResult, TagMapper::map); + } + + @Override + public AutoCompleteResults autoComplete(@Nonnull String query, + @Nullable String field, + @Nullable List filters, + int limit, + @Nonnull QueryContext context) throws Exception { + final Map facetFilters = ResolverUtils.buildFacetFilters(filters, TAG_SEARCH_CONFIG.getFacetFields()); + final AutoCompleteResult result = _tagClient.autocomplete(query, field, facetFilters, limit); + return AutoCompleteResultsMapper.map(result); + } + + + @Override + public Tag update(@Nonnull TagUpdate input, @Nonnull QueryContext context) throws Exception { + // TODO: Verify that updater is owner. + final CorpuserUrn actor = new CorpuserUrn(context.getActor()); + final com.linkedin.tag.Tag partialTag = TagUpdateMapper.map(input); + + // Create Audit Stamp + final AuditStamp auditStamp = new AuditStamp(); + auditStamp.setActor(actor, SetMode.IGNORE_NULL); + auditStamp.setTime(System.currentTimeMillis()); + + if (partialTag.hasOwnership()) { + partialTag.getOwnership().setLastModified(auditStamp); + } else { + final Ownership ownership = new Ownership(); + final Owner owner = new Owner(); + owner.setOwner(actor); + owner.setType(OwnershipType.DATAOWNER); + owner.setSource(new OwnershipSource().setType(OwnershipSourceType.SERVICE)); + + ownership.setOwners(new OwnerArray(owner)); + ownership.setLastModified(auditStamp); + partialTag.setOwnership(ownership); + } + + partialTag.setLastModified(auditStamp); + + try { + _tagClient.update(TagUrn.createFromString(input.getUrn()), partialTag); + } catch (RemoteInvocationException e) { + throw new RuntimeException(String.format("Failed to write entity with urn %s", input.getUrn()), e); + } + + return load(input.getUrn(), context); + } + private TagUrn getTagUrn(final String urnStr) { try { return TagUrn.createFromString(urnStr); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagAssociationUpdateMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagAssociationUpdateMapper.java new file mode 100644 index 0000000000..775c123070 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagAssociationUpdateMapper.java @@ -0,0 +1,30 @@ +package com.linkedin.datahub.graphql.types.tag.mappers; + +import com.linkedin.common.TagAssociation; +import com.linkedin.common.urn.TagUrn; +import com.linkedin.datahub.graphql.generated.TagAssociationUpdate; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; + +import javax.annotation.Nonnull; +import java.net.URISyntaxException; + +public class TagAssociationUpdateMapper implements ModelMapper { + + public static final TagAssociationUpdateMapper INSTANCE = new TagAssociationUpdateMapper(); + + public static TagAssociation map(@Nonnull final TagAssociationUpdate tagAssociationUpdate) { + return INSTANCE.apply(tagAssociationUpdate); + } + + public TagAssociation apply(final TagAssociationUpdate tagAssociationUpdate) { + final TagAssociation output = new TagAssociation(); + try { + output.setTag(TagUrn.createFromString(tagAssociationUpdate.getTag().getUrn())); + } catch (URISyntaxException e) { + throw new RuntimeException(String.format("Failed to update tag with urn %s, invalid urn", + tagAssociationUpdate.getTag().getUrn())); + } + return output; + } + +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagUpdateMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagUpdateMapper.java new file mode 100644 index 0000000000..d781e3c905 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagUpdateMapper.java @@ -0,0 +1,31 @@ +package com.linkedin.datahub.graphql.types.tag.mappers; + +import com.linkedin.common.urn.TagUrn; +import com.linkedin.datahub.graphql.generated.TagUpdate; +import com.linkedin.datahub.graphql.types.common.mappers.OwnershipUpdateMapper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; + +import javax.annotation.Nonnull; + +public class TagUpdateMapper implements ModelMapper { + + public static final TagUpdateMapper INSTANCE = new TagUpdateMapper(); + + public static com.linkedin.tag.Tag map(@Nonnull final TagUpdate tagUpdate) { + return INSTANCE.apply(tagUpdate); + } + + @Override + public com.linkedin.tag.Tag apply(@Nonnull final TagUpdate tagUpdate) { + final com.linkedin.tag.Tag result = new com.linkedin.tag.Tag(); + result.setUrn((new TagUrn(tagUpdate.getName()))); + result.setName(tagUpdate.getName()); + if (tagUpdate.getDescription() != null) { + result.setDescription(tagUpdate.getDescription()); + } + if (tagUpdate.getOwnership() != null) { + result.setOwnership(OwnershipUpdateMapper.map(tagUpdate.getOwnership())); + } + return result; + } +} diff --git a/datahub-graphql-core/src/main/resources/gms.graphql b/datahub-graphql-core/src/main/resources/gms.graphql index c59f3c0be6..c50f5aa5a2 100644 --- a/datahub-graphql-core/src/main/resources/gms.graphql +++ b/datahub-graphql-core/src/main/resources/gms.graphql @@ -108,6 +108,9 @@ type Query { type Mutation { updateDataset(input: DatasetUpdateInput!): Dataset + updateChart(input: ChartUpdateInput!): Chart + updateDashboard(input: DashboardUpdateInput!): Dashboard + updateTag(input: TagUpdate!): Tag } type AuditStamp { @@ -193,6 +196,11 @@ type Dataset implements Entity { """ schema: Schema + """ + Editable schema metadata of the dataset + """ + editableSchemaMetadata: EditableSchemaMetadata + """ Status metadata of the dataset """ @@ -473,7 +481,28 @@ type SchemaField { """ recursive: Boolean! """ - The structured tags associated with the field + Tags associated with the field + """ + globalTags: GlobalTags +} + +type EditableSchemaMetadata { + editableSchemaFieldInfo: [EditableSchemaFieldInfo!]! +} + +type EditableSchemaFieldInfo { + """ + Flattened name of a field identifying the field the editable info is applied to + """ + fieldPath: String! + + """ + Edited description of the field + """ + description: String + + """ + Tags associated with the field """ globalTags: GlobalTags } @@ -1052,6 +1081,30 @@ type BrowsePath { path: [String!]! } +input ChartUpdateInput { + """ + The chart urn + """ + urn: String! + + """ + Update to global tags + """ + globalTags: GlobalTagsUpdate +} + +input DashboardUpdateInput { + """ + The dashboard urn + """ + urn: String! + + """ + Update to global tags + """ + globalTags: GlobalTagsUpdate +} + input DatasetUpdateInput { """ The dataset urn @@ -1072,6 +1125,61 @@ input DatasetUpdateInput { Update to institutional memory, ie documentation """ institutionalMemory: InstitutionalMemoryUpdate + + """ + Update to global tags + """ + globalTags: GlobalTagsUpdate + + """ + Update to editable schema metadata of the dataset + """ + editableSchemaMetadata: EditableSchemaMetadataUpdate +} + +input EditableSchemaMetadataUpdate { + editableSchemaFieldInfo: [EditableSchemaFieldInfoUpdate!]! +} + +input EditableSchemaFieldInfoUpdate { + """ + Flattened name of a field identifying the field the editable info is applied to + """ + fieldPath: String! + + """ + Edited description of the field + """ + description: String + + """ + Tags associated with the field + """ + globalTags: GlobalTagsUpdate +} + +input GlobalTagsUpdate { + tags: [TagAssociationUpdate!] +} + +input TagAssociationUpdate { + tag: TagUpdate! +} + +input TagUpdate { + urn: String! + + name: String! + + """ + Description of the tag + """ + description: String + + """ + Ownership metadata of the tag + """ + ownership: OwnershipUpdate } input OwnershipUpdate { diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index dd4edf8f2e..3f0e1a0690 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -233,6 +233,7 @@ const dataset3 = { ], }, schema: null, + editableSchemaMetadata: null, deprecation: null, } as Dataset; @@ -677,6 +678,83 @@ export const mocks = [ }, }, }, + { + request: { + query: GetSearchResultsDocument, + variables: { + input: { + type: 'CORP_USER', + query: 'tags:abc-sample-tag', + start: 0, + count: 1, + filters: [], + }, + }, + }, + result: { + data: { + __typename: 'Query', + search: { + __typename: 'SearchResults', + start: 0, + count: 0, + total: 2, + entities: [], + facets: [], + }, + }, + }, + }, + { + request: { + query: GetSearchResultsDocument, + variables: { + input: { + type: 'DATASET', + query: 'tags:abc-sample-tag', + start: 0, + count: 1, + filters: [], + }, + }, + }, + result: { + data: { + __typename: 'Query', + search: { + __typename: 'SearchResults', + start: 0, + count: 1, + total: 1, + entities: [ + { + __typename: 'Dataset', + ...dataset3, + }, + ], + facets: [ + { + field: 'origin', + aggregations: [ + { + value: 'PROD', + count: 3, + }, + ], + }, + { + field: 'platform', + aggregations: [ + { value: 'hdfs', count: 1 }, + { value: 'mysql', count: 1 }, + { value: 'kafka', count: 1 }, + ], + }, + ], + }, + } as GetSearchResultsQuery, + }, + }, { request: { query: GetTagDocument, diff --git a/datahub-web-react/src/app/entity/chart/profile/ChartProfile.tsx b/datahub-web-react/src/app/entity/chart/profile/ChartProfile.tsx index 3c4f735971..a78733b682 100644 --- a/datahub-web-react/src/app/entity/chart/profile/ChartProfile.tsx +++ b/datahub-web-react/src/app/entity/chart/profile/ChartProfile.tsx @@ -5,9 +5,10 @@ import { Chart, GlobalTags } from '../../../../types.generated'; import { Ownership as OwnershipView } from '../../shared/Ownership'; import { EntityProfile } from '../../../shared/EntityProfile'; import ChartHeader from './ChartHeader'; -import { useGetChartQuery } from '../../../../graphql/chart.generated'; +import { GetChartDocument, useGetChartQuery, useUpdateChartMutation } from '../../../../graphql/chart.generated'; import ChartSources from './ChartSources'; import { Message } from '../../../shared/Message'; +import TagGroup from '../../../shared/tags/TagGroup'; const PageContainer = styled.div` padding: 32px 100px; @@ -22,6 +23,20 @@ const ENABLED_TAB_TYPES = [TabType.Ownership, TabType.Sources]; export default function ChartProfile({ urn }: { urn: string }) { const { loading, error, data } = useGetChartQuery({ variables: { urn } }); + const [updateChart] = useUpdateChartMutation({ + update(cache, { data: newChart }) { + cache.modify({ + fields: { + chart() { + cache.writeQuery({ + query: GetChartDocument, + data: { chart: { ...newChart?.updateChart } }, + }); + }, + }, + }); + }, + }); if (error || (!loading && !error && !data)) { return ; @@ -64,7 +79,14 @@ export default function ChartProfile({ urn }: { urn: string }) { {loading && } {data && data.chart && ( updateChart({ variables: { input: { urn, globalTags } } })} + /> + } title={data.chart.info?.name || ''} tabs={getTabs(data.chart as Chart)} header={getHeader(data.chart as Chart)} diff --git a/datahub-web-react/src/app/entity/dashboard/profile/DashboardProfile.tsx b/datahub-web-react/src/app/entity/dashboard/profile/DashboardProfile.tsx index 816c4094be..fc4fe03976 100644 --- a/datahub-web-react/src/app/entity/dashboard/profile/DashboardProfile.tsx +++ b/datahub-web-react/src/app/entity/dashboard/profile/DashboardProfile.tsx @@ -1,13 +1,18 @@ import { Alert } from 'antd'; import React from 'react'; import styled from 'styled-components'; -import { useGetDashboardQuery } from '../../../../graphql/dashboard.generated'; +import { + GetDashboardDocument, + useGetDashboardQuery, + useUpdateDashboardMutation, +} from '../../../../graphql/dashboard.generated'; import { Dashboard, GlobalTags } from '../../../../types.generated'; import { Ownership as OwnershipView } from '../../shared/Ownership'; import { EntityProfile } from '../../../shared/EntityProfile'; import DashboardHeader from './DashboardHeader'; import DashboardCharts from './DashboardCharts'; import { Message } from '../../../shared/Message'; +import TagGroup from '../../../shared/tags/TagGroup'; const PageContainer = styled.div` padding: 32px 100px; @@ -25,6 +30,20 @@ const ENABLED_TAB_TYPES = [TabType.Ownership, TabType.Charts]; */ export default function DashboardProfile({ urn }: { urn: string }) { const { loading, error, data } = useGetDashboardQuery({ variables: { urn } }); + const [updateDashboard] = useUpdateDashboardMutation({ + update(cache, { data: newDashboard }) { + cache.modify({ + fields: { + dashboard() { + cache.writeQuery({ + query: GetDashboardDocument, + data: { dashboard: { ...newDashboard?.updateDashboard } }, + }); + }, + }, + }); + }, + }); if (error || (!loading && !error && !data)) { return ; @@ -68,7 +87,16 @@ export default function DashboardProfile({ urn }: { urn: string }) { {data && data.dashboard && ( + updateDashboard({ variables: { input: { urn, globalTags } } }) + } + /> + } tabs={getTabs(data.dashboard as Dashboard)} header={getHeader(data.dashboard as Dashboard)} /> diff --git a/datahub-web-react/src/app/entity/dataset/profile/DatasetProfile.tsx b/datahub-web-react/src/app/entity/dataset/profile/DatasetProfile.tsx index 12b76a2701..64552740fd 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/DatasetProfile.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/DatasetProfile.tsx @@ -1,6 +1,10 @@ import React from 'react'; import { Alert } from 'antd'; -import { useGetDatasetQuery, useUpdateDatasetMutation } from '../../../../graphql/dataset.generated'; +import { + useGetDatasetQuery, + useUpdateDatasetMutation, + GetDatasetDocument, +} from '../../../../graphql/dataset.generated'; import { Ownership as OwnershipView } from '../../shared/Ownership'; import SchemaView from './schema/Schema'; import { EntityProfile } from '../../../shared/EntityProfile'; @@ -10,6 +14,7 @@ import PropertiesView from './Properties'; import DocumentsView from './Documentation'; import DatasetHeader from './DatasetHeader'; import { Message } from '../../../shared/Message'; +import TagGroup from '../../../shared/tags/TagGroup'; export enum TabType { Ownership = 'Ownership', @@ -27,7 +32,20 @@ const EMPTY_ARR: never[] = []; */ export const DatasetProfile = ({ urn }: { urn: string }): JSX.Element => { const { loading, error, data } = useGetDatasetQuery({ variables: { urn } }); - const [updateDataset] = useUpdateDatasetMutation(); + const [updateDataset] = useUpdateDatasetMutation({ + update(cache, { data: newDataset }) { + cache.modify({ + fields: { + dataset() { + cache.writeQuery({ + query: GetDatasetDocument, + data: { dataset: newDataset?.updateDataset }, + }); + }, + }, + }); + }, + }); if (error || (!loading && !error && !data)) { return ; @@ -42,12 +60,21 @@ export const DatasetProfile = ({ urn }: { urn: string }): JSX.Element => { properties, institutionalMemory, schema, + editableSchemaMetadata, }: Dataset) => { return [ { name: TabType.Schema, path: TabType.Schema.toLowerCase(), - content: , + content: ( + + updateDataset({ variables: { input: { urn, editableSchemaMetadata: update } } }) + } + /> + ), }, { name: TabType.Ownership, @@ -93,7 +120,14 @@ export const DatasetProfile = ({ urn }: { urn: string }): JSX.Element => { {data && data.dataset && ( updateDataset({ variables: { input: { urn, globalTags } } })} + /> + } tabs={getTabs(data.dataset as Dataset)} header={getHeader(data.dataset as Dataset)} /> diff --git a/datahub-web-react/src/app/entity/dataset/profile/__tests__/Schema.test.tsx b/datahub-web-react/src/app/entity/dataset/profile/__tests__/Schema.test.tsx index b5f7035aab..c74aff4eed 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/__tests__/Schema.test.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/__tests__/Schema.test.tsx @@ -2,13 +2,13 @@ import React from 'react'; import { fireEvent, render } from '@testing-library/react'; import Schema from '../schema/Schema'; import TestPageContainer from '../../../../../utils/test-utils/TestPageContainer'; -import { sampleSchema } from '../stories/sampleSchema'; +import { sampleSchema, sampleSchemaWithTags } from '../stories/sampleSchema'; describe('Schema', () => { it('renders', () => { const { getByText, queryAllByTestId } = render( - + , ); expect(getByText('name')).toBeInTheDocument(); @@ -21,7 +21,7 @@ describe('Schema', () => { it('renders raw', () => { const { getByText, queryAllByTestId } = render( - + , ); @@ -40,4 +40,13 @@ describe('Schema', () => { expect(queryAllByTestId('icon-STRING')).toHaveLength(2); expect(queryAllByTestId('schema-raw-view')).toHaveLength(0); }); + + it('renders tags', () => { + const { getByText } = render( + + + , + ); + expect(getByText('Legacy')).toBeInTheDocument(); + }); }); diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/Schema.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/Schema.tsx index d558258d31..a4b3b2fb1c 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/Schema.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/Schema.tsx @@ -1,12 +1,25 @@ -import React, { useMemo, useState } from 'react'; +import React, { useState } from 'react'; import { Button, Table, Typography } from 'antd'; import { AlignType } from 'rc-table/lib/interface'; import styled from 'styled-components'; +import { FetchResult } from '@apollo/client'; import TypeIcon from './TypeIcon'; -import { Schema, SchemaFieldDataType, GlobalTags } from '../../../../../types.generated'; -import TagGroup from '../../../../shared/TagGroup'; +import { + Schema, + SchemaFieldDataType, + GlobalTags, + EditableSchemaMetadata, + SchemaField, + EditableSchemaMetadataUpdate, + GlobalTagsUpdate, + EditableSchemaFieldInfo, + EditableSchemaFieldInfoUpdate, +} from '../../../../../types.generated'; +import TagGroup from '../../../../shared/tags/TagGroup'; +import { UpdateDatasetMutation } from '../../../../../graphql/dataset.generated'; +import { convertTagsForUpdate } from '../../../../shared/tags/utils/convertTagsForUpdate'; const ViewRawButtonContainer = styled.div` display: flex; @@ -16,6 +29,10 @@ const ViewRawButtonContainer = styled.div` export type Props = { schema?: Schema | null; + editableSchemaMetadata?: EditableSchemaMetadata | null; + updateEditableSchema: ( + update: EditableSchemaMetadataUpdate, + ) => Promise, Record>>; }; const defaultColumns = [ @@ -44,21 +61,82 @@ const defaultColumns = [ }, ]; -const tagColumn = { - title: 'Tags', - dataIndex: 'globalTags', - key: 'tag', - render: (tags: GlobalTags) => { - return ; - }, -}; +function convertEditableSchemaMetadataForUpdate( + editableSchemaMetadata: EditableSchemaMetadata | null | undefined, +): EditableSchemaMetadataUpdate { + return { + editableSchemaFieldInfo: + editableSchemaMetadata?.editableSchemaFieldInfo.map((editableSchemaFieldInfo) => ({ + fieldPath: editableSchemaFieldInfo?.fieldPath, + description: editableSchemaFieldInfo?.description, + globalTags: { tags: convertTagsForUpdate(editableSchemaFieldInfo?.globalTags?.tags || []) }, + })) || [], + }; +} -export default function SchemaView({ schema }: Props) { - const columns = useMemo(() => { - const hasTags = schema?.fields?.some((field) => (field?.globalTags?.tags?.length || 0) > 0); +export default function SchemaView({ schema, editableSchemaMetadata, updateEditableSchema }: Props) { + const [hoveredIndex, setHoveredIndex] = useState(undefined); - return [...defaultColumns, ...(hasTags ? [tagColumn] : [])]; - }, [schema]); + const onUpdateTags = (update: GlobalTagsUpdate, record?: EditableSchemaFieldInfo) => { + if (!record) return Promise.resolve(); + const newFieldInfo: EditableSchemaFieldInfoUpdate = { + fieldPath: record?.fieldPath, + description: record?.description, + globalTags: update, + }; + + let existingMetadataAsUpdate = convertEditableSchemaMetadataForUpdate(editableSchemaMetadata); + + if (existingMetadataAsUpdate.editableSchemaFieldInfo.some((field) => field.fieldPath === record?.fieldPath)) { + // if we already have a record for this field, update the record + existingMetadataAsUpdate = { + editableSchemaFieldInfo: existingMetadataAsUpdate.editableSchemaFieldInfo.map((fieldUpdate) => { + if (fieldUpdate.fieldPath === record?.fieldPath) { + return newFieldInfo; + } + return fieldUpdate; + }), + }; + } else { + // otherwise add a new record + existingMetadataAsUpdate.editableSchemaFieldInfo.push(newFieldInfo); + } + return updateEditableSchema(existingMetadataAsUpdate); + }; + + const tagGroupRender = (tags: GlobalTags, record: SchemaField, rowIndex: number | undefined) => { + const relevantEditableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo.find( + (candidateEditableFieldInfo) => candidateEditableFieldInfo.fieldPath === record.fieldPath, + ); + return ( + setHoveredIndex(undefined)} + updateTags={(update) => + onUpdateTags(update, relevantEditableFieldInfo || { fieldPath: record.fieldPath }) + } + /> + ); + }; + + const tagColumn = { + width: 450, + title: 'Tags', + dataIndex: 'globalTags', + key: 'tag', + render: tagGroupRender, + onCell: (record: SchemaField, rowIndex: number | undefined) => ({ + onMouseEnter: () => { + setHoveredIndex(rowIndex); + }, + onMouseLeave: () => { + setHoveredIndex(undefined); + }, + }), + }; const [showRaw, setShowRaw] = useState(false); @@ -79,7 +157,12 @@ export default function SchemaView({ schema }: Props) { ) : ( - +
)} ); diff --git a/datahub-web-react/src/app/entity/dataset/profile/stories/sampleSchema.ts b/datahub-web-react/src/app/entity/dataset/profile/stories/sampleSchema.ts index 83b2204df4..633c3d5928 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/stories/sampleSchema.ts +++ b/datahub-web-react/src/app/entity/dataset/profile/stories/sampleSchema.ts @@ -1,20 +1,10 @@ -import { volcano, lime, geekblue, purple, gold, yellow } from '@ant-design/colors'; -import { Schema, SchemaField, SchemaFieldDataType } from '../../../../../types.generated'; +import { EntityType, Schema, SchemaField, SchemaFieldDataType } from '../../../../../types.generated'; // Extending the schema type with an option for tags export type TaggedSchemaField = { tags: Tag[]; } & SchemaField; -const TAGS = { - pii: { name: 'PII', color: volcano[1], category: 'Privacy' }, - financial: { name: 'Financial', color: gold[1], category: 'Privacy' }, - address: { name: 'Address', color: lime[1], category: 'Descriptor', descriptor: true }, - shipping: { name: 'Shipping', color: yellow[1], category: 'Privacy' }, - name: { name: 'Name', color: purple[1], category: 'Descriptor', descriptor: true }, - euro: { name: 'Currency', value: 'Euros', color: geekblue[1], category: 'Descriptor', descriptor: true }, -}; - export type Tag = { name: string; value?: string; @@ -114,6 +104,18 @@ export const sampleSchemaWithTags: Schema = { type: SchemaFieldDataType.Number, nativeDataType: 'number', recursive: false, + globalTags: { + tags: [ + { + tag: { + urn: 'urn:li:tag:Legacy', + name: 'Legacy', + description: 'this is a legacy dataset', + type: EntityType.Tag, + }, + }, + ], + }, }, { fieldPath: 'name', @@ -122,7 +124,6 @@ export const sampleSchemaWithTags: Schema = { type: SchemaFieldDataType.String, nativeDataType: 'string', recursive: false, - tags: [TAGS.name, TAGS.pii], } as SchemaField, { fieldPath: 'shipping_address', @@ -131,7 +132,6 @@ export const sampleSchemaWithTags: Schema = { type: SchemaFieldDataType.String, nativeDataType: 'string', recursive: false, - tags: [TAGS.address, TAGS.pii, TAGS.shipping], } as SchemaField, { fieldPath: 'count', @@ -148,7 +148,6 @@ export const sampleSchemaWithTags: Schema = { type: SchemaFieldDataType.Number, nativeDataType: 'number', recursive: false, - tags: [TAGS.euro], } as SchemaField, { fieldPath: 'was_returned', @@ -173,7 +172,6 @@ export const sampleSchemaWithTags: Schema = { type: SchemaFieldDataType.Struct, nativeDataType: 'struct', recursive: false, - tags: [TAGS.financial], } as SchemaField, ], }; diff --git a/datahub-web-react/src/app/entity/tag/TagProfile.tsx b/datahub-web-react/src/app/entity/tag/TagProfile.tsx index 8e6aff5ca9..8bca1345cb 100644 --- a/datahub-web-react/src/app/entity/tag/TagProfile.tsx +++ b/datahub-web-react/src/app/entity/tag/TagProfile.tsx @@ -1,13 +1,15 @@ import { grey } from '@ant-design/colors'; -import { Alert, Avatar, Card, Space, Tooltip, Typography } from 'antd'; +import { Alert, Avatar, Button, Card, Tooltip, Typography } from 'antd'; import React from 'react'; -import { useParams } from 'react-router'; +import { useHistory, useParams } from 'react-router'; import { Link } from 'react-router-dom'; import styled from 'styled-components'; import { useGetTagQuery } from '../../../graphql/tag.generated'; import defaultAvatar from '../../../images/default_avatar.png'; import { EntityType } from '../../../types.generated'; +import { useGetAllEntitySearchResults } from '../../../utils/customGraphQL/useGetAllEntitySearchResults'; +import { navigateToSearchUrl } from '../../search/utils/navigateToSearchUrl'; import { Message } from '../../shared/Message'; import { useEntityRegistry } from '../../useEntityRegistry'; @@ -22,7 +24,21 @@ const LoadingMessage = styled(Message)` const TitleLabel = styled(Typography.Text)` &&& { color: ${grey[2]}; - font-size: 13; + font-size: 13px; + } +`; + +const CreatedByLabel = styled(Typography.Text)` + &&& { + color: ${grey[2]}; + font-size: 13px; + } +`; + +const StatsLabel = styled(Typography.Text)` + &&& { + color: ${grey[2]}; + font-size: 13px; } `; @@ -32,6 +48,29 @@ const TitleText = styled(Typography.Title)` } `; +const HeaderLayout = styled.div` + display: flex; + justify-content: space-between; +`; + +const StatsBox = styled.div` + width: 180px; + justify-content: left; +`; + +const StatText = styled(Typography.Text)` + font-size: 15px; +`; + +const EmptyStatsText = styled(Typography.Text)` + font-size: 15px; + font-style: italic; +`; + +const TagSearchButton = styled(Button)` + margin-left: -16px; +`; + type TagPageParams = { urn: string; }; @@ -43,6 +82,24 @@ export default function TagProfile() { const { urn } = useParams(); const { loading, error, data } = useGetTagQuery({ variables: { urn } }); const entityRegistry = useEntityRegistry(); + const history = useHistory(); + + const allSearchResultsByType = useGetAllEntitySearchResults({ + query: `tags:${data?.tag?.name}`, + start: 0, + count: 1, + filters: [], + }); + + const statsLoading = Object.keys(allSearchResultsByType).some((type) => { + return allSearchResultsByType[type].loading; + }); + + const someStats = + !statsLoading && + Object.keys(allSearchResultsByType).some((type) => { + return allSearchResultsByType[type]?.data.search.total > 0; + }); if (error || (!loading && !error && !data)) { return ; @@ -53,30 +110,76 @@ export default function TagProfile() { {loading && } - + +
Tag {data?.tag?.name}
- - {data?.tag?.ownership?.owners?.map((owner) => ( - - - - - - ))} - - - +
+
+ Created by +
+ + {data?.tag?.ownership?.owners?.map((owner) => ( + + + + + + ))} + +
+
+ + Applied to + {statsLoading && ( +
+ Loading... +
+ )} + {!statsLoading && !someStats && ( +
+ No entities +
+ )} + {!statsLoading && + someStats && + Object.keys(allSearchResultsByType).map((type) => { + if (allSearchResultsByType[type].data.search.total === 0) { + return null; + } + return ( +
+ + navigateToSearchUrl({ + type: type as EntityType, + query: `tags:${data?.tag?.name}`, + history, + entityRegistry, + }) + } + > + + {allSearchResultsByType[type].data.search.total}{' '} + {entityRegistry.getCollectionName(type as EntityType)} + + +
+ ); + })} +
+
} > diff --git a/datahub-web-react/src/app/entity/tag/__tests__/TagProfile.test.tsx b/datahub-web-react/src/app/entity/tag/__tests__/TagProfile.test.tsx index 428b8f76a5..6fe3e5ac4c 100644 --- a/datahub-web-react/src/app/entity/tag/__tests__/TagProfile.test.tsx +++ b/datahub-web-react/src/app/entity/tag/__tests__/TagProfile.test.tsx @@ -44,4 +44,23 @@ describe('TagProfile', () => { 'http://localhost/user/urn:li:corpuser:3', ); }); + + it('renders stats', async () => { + const { getByTestId, queryByText } = render( + + + } /> + + , + ); + + await waitFor(() => expect(queryByText('abc-sample-tag')).toBeInTheDocument()); + + await waitFor(() => expect(queryByText('Loading')).not.toBeInTheDocument()); + + expect(getByTestId('stats-DATASET')).toBeInTheDocument(); + expect(getByTestId('stats-CORP_USER')).toBeInTheDocument(); + expect(queryByText('1 Datasets')).toBeInTheDocument(); + expect(queryByText('2 Users')).toBeInTheDocument(); + }); }); diff --git a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx index eea45a9626..61061d2bb5 100644 --- a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx +++ b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx @@ -4,7 +4,7 @@ import { Link } from 'react-router-dom'; import { EntityType, GlobalTags } from '../../types.generated'; import defaultAvatar from '../../images/default_avatar.png'; import { useEntityRegistry } from '../useEntityRegistry'; -import TagGroup from '../shared/TagGroup'; +import TagGroup from '../shared/tags/TagGroup'; interface Props { name: string; @@ -70,11 +70,7 @@ export default function DefaultPreviewCard({ )}
- {tags && tags.tags?.length && ( - - - - )} + {tags && tags.tags?.length && } Owned By diff --git a/datahub-web-react/src/app/shared/EntityProfile.tsx b/datahub-web-react/src/app/shared/EntityProfile.tsx index 9b917ac294..539c03e17a 100644 --- a/datahub-web-react/src/app/shared/EntityProfile.tsx +++ b/datahub-web-react/src/app/shared/EntityProfile.tsx @@ -1,14 +1,13 @@ import * as React from 'react'; -import { Col, Row, Divider, Layout, Space } from 'antd'; +import { Col, Row, Divider, Layout, Card, Typography } from 'antd'; import styled from 'styled-components'; +import { TagOutlined } from '@ant-design/icons'; import { RoutedTabs } from './RoutedTabs'; -import { GlobalTags } from '../../types.generated'; -import TagGroup from './TagGroup'; export interface EntityProfileProps { title: string; - tags?: GlobalTags; + tags?: React.ReactNode; header: React.ReactNode; tabs?: Array<{ name: string; @@ -17,8 +16,24 @@ export interface EntityProfileProps { }>; } -const TagsContainer = styled.div` - margin-top: -8px; +const FlexSpace = styled.div` + display: flex; + justify-content: space-between; +`; + +const TagsTitle = styled(Typography.Title)` + font-size: 18px; +`; + +const TagCard = styled(Card)` + margin-top: 24px; + font-size: 18px; + width: 450px; + height: 100%; +`; + +const TagIcon = styled(TagOutlined)` + padding-right: 6px; `; const defaultProps = { @@ -35,23 +50,30 @@ export const EntityProfile = ({ title, tags, header, tabs }: EntityProfileProps) /* eslint-disable spaced-comment */ return ( - -
- -

{title}

- - - -
- - - {header} - - - - - - +
+ +
+ +
+

{title}

+ + + {header} + + + + Tags + + {tags} + + + + + + + + + ); }; diff --git a/datahub-web-react/src/app/shared/TagGroup.tsx b/datahub-web-react/src/app/shared/TagGroup.tsx deleted file mode 100644 index 6432704bea..0000000000 --- a/datahub-web-react/src/app/shared/TagGroup.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import { Space, Tag } from 'antd'; -import React from 'react'; -import { Link } from 'react-router-dom'; -import { useEntityRegistry } from '../useEntityRegistry'; -import { EntityType, GlobalTags } from '../../types.generated'; - -type Props = { - globalTags?: GlobalTags | null; -}; - -export default function TagGroup({ globalTags }: Props) { - const entityRegistry = useEntityRegistry(); - - return ( - - {globalTags?.tags?.map((tag) => ( - - {tag.tag.name} - - ))} - - ); -} diff --git a/datahub-web-react/src/app/shared/tags/AddTagModal.tsx b/datahub-web-react/src/app/shared/tags/AddTagModal.tsx new file mode 100644 index 0000000000..bbd9e4c887 --- /dev/null +++ b/datahub-web-react/src/app/shared/tags/AddTagModal.tsx @@ -0,0 +1,130 @@ +import React, { useState } from 'react'; +import { FetchResult } from '@apollo/client'; +import { Button, Modal, Select, Typography } from 'antd'; +import styled from 'styled-components'; + +import { UpdateDatasetMutation } from '../../../graphql/dataset.generated'; +import { useGetAutoCompleteResultsLazyQuery } from '../../../graphql/search.generated'; +import { GlobalTags, GlobalTagsUpdate, EntityType, TagAssociationUpdate } from '../../../types.generated'; +import { convertTagsForUpdate } from './utils/convertTagsForUpdate'; +import CreateTagModal from './CreateTagModal'; + +type AddTagModalProps = { + globalTags?: GlobalTags | null; + updateTags?: ( + update: GlobalTagsUpdate, + ) => Promise, Record>>; + visible: boolean; + onClose: () => void; +}; + +const TagSelect = styled(Select)` + width: 480px; +`; + +const CREATE_TAG_VALUE = '____reserved____.createTagValue'; + +export default function AddTagModal({ updateTags, globalTags, visible, onClose }: AddTagModalProps) { + const [getAutoCompleteResults, { loading, data: suggestionsData }] = useGetAutoCompleteResultsLazyQuery(); + const [inputValue, setInputValue] = useState(''); + const [selectedTagValue, setSelectedTagValue] = useState(''); + const [showCreateModal, setShowCreateModal] = useState(false); + const [disableAdd, setDisableAdd] = useState(false); + + const autoComplete = (query: string) => { + getAutoCompleteResults({ + variables: { + input: { + type: EntityType.Tag, + query, + }, + }, + }); + }; + + const inputExistsInAutocomplete = suggestionsData?.autoComplete?.suggestions?.some( + (result) => result.toLowerCase() === inputValue.toLowerCase(), + ); + + const autocompleteOptions = + suggestionsData?.autoComplete?.suggestions.map((result) => ( + + {result} + + )) || []; + + if (!inputExistsInAutocomplete && inputValue.length > 2 && !loading) { + autocompleteOptions.push( + + Create {inputValue} + , + ); + } + + const onOk = () => { + if (!globalTags?.tags?.some((tag) => tag.tag.name === selectedTagValue)) { + setDisableAdd(true); + updateTags?.({ + tags: [ + ...convertTagsForUpdate(globalTags?.tags || []), + { tag: { urn: `urn:li:tag:${selectedTagValue}`, name: selectedTagValue } }, + ] as TagAssociationUpdate[], + }).finally(() => { + setDisableAdd(false); + onClose(); + }); + } else { + onClose(); + } + }; + + if (showCreateModal) { + return ( + setShowCreateModal(false)} + tagName={inputValue} + /> + ); + } + + return ( + + + + + } + > + { + autoComplete(value); + setInputValue(value); + }} + onSelect={(selected) => + selected === CREATE_TAG_VALUE ? setShowCreateModal(true) : setSelectedTagValue(String(selected)) + } + notFoundContent={loading ? 'loading' : 'type at least 3 character to search'} + > + {autocompleteOptions} + + + ); +} diff --git a/datahub-web-react/src/app/shared/tags/CreateTagModal.tsx b/datahub-web-react/src/app/shared/tags/CreateTagModal.tsx new file mode 100644 index 0000000000..9db97d7437 --- /dev/null +++ b/datahub-web-react/src/app/shared/tags/CreateTagModal.tsx @@ -0,0 +1,87 @@ +import React, { useState } from 'react'; +import { FetchResult } from '@apollo/client'; +import { Button, Input, Modal, Space } from 'antd'; +import styled from 'styled-components'; + +import { UpdateDatasetMutation } from '../../../graphql/dataset.generated'; +import { useUpdateTagMutation } from '../../../graphql/tag.generated'; +import { GlobalTags, GlobalTagsUpdate, TagAssociationUpdate } from '../../../types.generated'; +import { convertTagsForUpdate } from './utils/convertTagsForUpdate'; + +type CreateTagModalProps = { + globalTags?: GlobalTags | null; + updateTags?: ( + update: GlobalTagsUpdate, + ) => Promise, Record>>; + visible: boolean; + onClose: () => void; + onBack: () => void; + tagName: string; +}; + +const FullWidthSpace = styled(Space)` + width: 100%; +`; + +export default function CreateTagModal({ + updateTags, + globalTags, + onClose, + onBack, + visible, + tagName, +}: CreateTagModalProps) { + const [stagedDescription, setStagedDescription] = useState(''); + + const [updateTagMutation] = useUpdateTagMutation(); + const [disableCreate, setDisableCreate] = useState(false); + + const onOk = () => { + setDisableCreate(true); + // first create the new tag + updateTagMutation({ + variables: { + input: { + urn: `urn:li:tag:${tagName}`, + name: tagName, + description: stagedDescription, + }, + }, + }).then(() => { + // then apply the tag to the dataset + updateTags?.({ + tags: [ + ...convertTagsForUpdate(globalTags?.tags || []), + { tag: { urn: `urn:li:tag:${tagName}`, name: tagName } }, + ] as TagAssociationUpdate[], + }).finally(() => { + // and finally close the modal + setDisableCreate(false); + onClose(); + }); + }); + }; + + return ( + + + + + } + > + + setStagedDescription(e.target.value)} + /> + + + ); +} diff --git a/datahub-web-react/src/app/shared/tags/TagGroup.tsx b/datahub-web-react/src/app/shared/tags/TagGroup.tsx new file mode 100644 index 0000000000..774125032a --- /dev/null +++ b/datahub-web-react/src/app/shared/tags/TagGroup.tsx @@ -0,0 +1,109 @@ +import { Modal, Tag } from 'antd'; +import React, { useState } from 'react'; +import { Link } from 'react-router-dom'; +import styled from 'styled-components'; + +import { useEntityRegistry } from '../../useEntityRegistry'; +import { EntityType, GlobalTags, GlobalTagsUpdate } from '../../../types.generated'; +import { convertTagsForUpdate } from './utils/convertTagsForUpdate'; +import AddTagModal from './AddTagModal'; + +type Props = { + uneditableTags?: GlobalTags | null; + editableTags?: GlobalTags | null; + canRemove?: boolean; + canAdd?: boolean; + updateTags?: (update: GlobalTagsUpdate) => Promise; + onOpenModal?: () => void; + maxShow?: number; +}; + +const AddNewTag = styled(Tag)` + cursor: pointer; +`; + +export default function TagGroup({ + uneditableTags, + editableTags, + canRemove, + canAdd, + updateTags, + onOpenModal, + maxShow, +}: Props) { + const entityRegistry = useEntityRegistry(); + const [showAddModal, setShowAddModal] = useState(false); + + const removeTag = (urnToRemove: string) => { + onOpenModal?.(); + const tagToRemove = editableTags?.tags?.find((tag) => tag.tag.urn === urnToRemove); + const newTags = editableTags?.tags?.filter((tag) => tag.tag.urn !== urnToRemove); + Modal.confirm({ + title: `Do you want to remove ${tagToRemove?.tag.name} tag?`, + content: `Are you sure you want to remove the ${tagToRemove?.tag.name} tag?`, + onOk() { + updateTags?.({ tags: convertTagsForUpdate(newTags || []) }); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + }; + + let renderedTags = 0; + + return ( +
+ {/* uneditable tags are provided by ingestion pipelines exclusively */} + {uneditableTags?.tags?.map((tag) => { + renderedTags += 1; + if (maxShow && renderedTags > maxShow) return null; + return ( + + + {tag.tag.name} + + + ); + })} + {/* editable tags may be provided by ingestion pipelines or the UI */} + {editableTags?.tags?.map((tag) => { + renderedTags += 1; + if (maxShow && renderedTags > maxShow) return null; + return ( + + { + e.preventDefault(); + removeTag(tag.tag.urn); + }} + > + {tag.tag.name} + + + ); + })} + {canAdd && (uneditableTags?.tags?.length || 0) + (editableTags?.tags?.length || 0) < 10 && ( + <> + setShowAddModal(true)}> + + Add Tag + + {showAddModal && ( + { + onOpenModal?.(); + setShowAddModal(false); + }} + /> + )} + + )} +
+ ); +} diff --git a/datahub-web-react/src/app/shared/tags/__tests__/TagGroup.test.tsx b/datahub-web-react/src/app/shared/tags/__tests__/TagGroup.test.tsx new file mode 100644 index 0000000000..d32c9d18c2 --- /dev/null +++ b/datahub-web-react/src/app/shared/tags/__tests__/TagGroup.test.tsx @@ -0,0 +1,85 @@ +import React from 'react'; +import { fireEvent, render, waitFor } from '@testing-library/react'; +import TagGroup from '../TagGroup'; +import TestPageContainer from '../../../../utils/test-utils/TestPageContainer'; +import { EntityType } from '../../../../types.generated'; + +const legacyTag = { + urn: 'urn:li:tag:legacy', + name: 'Legacy', + description: 'this element is outdated', + type: EntityType.Tag, +}; + +const ownershipTag = { + urn: 'urn:li:tag:NeedsOwnership', + name: 'NeedsOwnership', + description: 'this element needs an owner', + type: EntityType.Tag, +}; + +const globalTags1 = { + tags: [{ tag: legacyTag }], +}; + +const globalTags2 = { + tags: [{ tag: ownershipTag }], +}; + +describe('TagGroup', () => { + it('renders editable tags', async () => { + const { getByText, getByLabelText, queryAllByLabelText, queryByText } = render( + + + , + ); + expect(queryByText('Add Tag')).not.toBeInTheDocument(); + expect(getByText('Legacy')).toBeInTheDocument(); + expect(queryAllByLabelText('close')).toHaveLength(1); + fireEvent.click(getByLabelText('close')); + await waitFor(() => expect(getByText('Do you want to remove Legacy tag?')).toBeInTheDocument()); + expect(getByText('Do you want to remove Legacy tag?')).toBeInTheDocument(); + + fireEvent.click(getByLabelText('Close')); + + await waitFor(() => expect(queryByText('Do you want to remove Legacy tag?')).not.toBeInTheDocument()); + + expect(getByText('Legacy')).toBeInTheDocument(); + }); + + it('renders uneditable tags', () => { + const { getByText, queryByLabelText, queryByText } = render( + + + , + ); + expect(queryByText('Add Tag')).not.toBeInTheDocument(); + expect(getByText('NeedsOwnership')).toBeInTheDocument(); + expect(queryByLabelText('close')).not.toBeInTheDocument(); + }); + + it('renders both together', () => { + const { getByText, queryByText, queryAllByLabelText } = render( + + + , + ); + expect(queryByText('Add Tag')).not.toBeInTheDocument(); + expect(getByText('Legacy')).toBeInTheDocument(); + expect(getByText('NeedsOwnership')).toBeInTheDocument(); + expect(queryAllByLabelText('close')).toHaveLength(1); + }); + + it('renders create tag', () => { + const { getByText, queryByText } = render( + + + , + ); + expect(queryByText('+ Add Tag')).toBeInTheDocument(); + expect(queryByText('Find a tag')).not.toBeInTheDocument(); + const AddTagButton = getByText('+ Add Tag'); + fireEvent.click(AddTagButton); + expect(queryByText('Find a tag')).toBeInTheDocument(); + }); +}); diff --git a/datahub-web-react/src/app/shared/tags/utils/convertTagsForUpdate.ts b/datahub-web-react/src/app/shared/tags/utils/convertTagsForUpdate.ts new file mode 100644 index 0000000000..c40f3a4244 --- /dev/null +++ b/datahub-web-react/src/app/shared/tags/utils/convertTagsForUpdate.ts @@ -0,0 +1,7 @@ +import { TagAssociation, TagAssociationUpdate } from '../../../../types.generated'; + +export function convertTagsForUpdate(tags: TagAssociation[]): TagAssociationUpdate[] { + return tags.map((tag) => ({ + tag: { urn: tag.tag.urn, name: tag.tag.name, description: tag.tag.description }, + })); +} diff --git a/datahub-web-react/src/graphql/chart.graphql b/datahub-web-react/src/graphql/chart.graphql index 40a7f759e4..1ee3d343cf 100644 --- a/datahub-web-react/src/graphql/chart.graphql +++ b/datahub-web-react/src/graphql/chart.graphql @@ -96,3 +96,102 @@ query getChart($urn: String!) { } } } + +mutation updateChart($input: ChartUpdateInput!) { + updateChart(input: $input) { + urn + type + tool + chartId + info { + name + description + inputs { + urn + name + origin + description + platform { + name + info { + logoUrl + } + } + platformNativeType + tags + ownership { + owners { + owner { + urn + type + username + info { + active + displayName + title + firstName + lastName + fullName + } + editableInfo { + pictureLink + } + } + type + } + lastModified { + time + } + } + } + url + type + access + lastRefreshed + lastModified { + time + } + created { + time + } + } + query { + rawQuery + type + } + ownership { + owners { + owner { + urn + type + username + info { + active + displayName + title + email + firstName + lastName + fullName + } + editableInfo { + pictureLink + } + } + type + } + lastModified { + time + } + } + globalTags { + tags { + tag { + urn + name + description + } + } + } + } +} diff --git a/datahub-web-react/src/graphql/dashboard.graphql b/datahub-web-react/src/graphql/dashboard.graphql index dd3129c90f..e5ccee2054 100644 --- a/datahub-web-react/src/graphql/dashboard.graphql +++ b/datahub-web-react/src/graphql/dashboard.graphql @@ -87,3 +87,93 @@ query getDashboard($urn: String!) { } } } + +mutation updateDashboard($input: DashboardUpdateInput!) { + updateDashboard(input: $input) { + urn + type + tool + dashboardId + info { + name + description + charts { + urn + tool + type + info { + name + description + } + ownership { + owners { + owner { + urn + type + username + info { + active + displayName + title + email + firstName + lastName + fullName + } + editableInfo { + pictureLink + } + } + type + } + lastModified { + time + } + } + } + url + access + lastRefreshed + created { + time + } + lastModified { + time + } + } + ownership { + owners { + owner { + urn + type + username + info { + active + displayName + title + email + firstName + lastName + fullName + } + editableInfo { + pictureLink + } + } + type + } + lastModified { + time + } + } + globalTags { + tags { + tag { + urn + name + description + } + } + } + } +} diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index 4f7e4bd03d..4d1acfbeeb 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -88,6 +88,21 @@ fragment nonRecursiveDatasetFields on Dataset { } primaryKeys } + editableSchemaMetadata { + editableSchemaFieldInfo { + fieldPath + description + globalTags { + tags { + tag { + urn + name + description + } + } + } + } + } deprecation { actor deprecated @@ -199,6 +214,21 @@ query getDataset($urn: String!) { } primaryKeys } + editableSchemaMetadata { + editableSchemaFieldInfo { + fieldPath + description + globalTags { + tags { + tag { + urn + name + description + } + } + } + } + } deprecation { actor deprecated diff --git a/datahub-web-react/src/graphql/tag.graphql b/datahub-web-react/src/graphql/tag.graphql index d3c9d28994..78dfedd5e5 100644 --- a/datahub-web-react/src/graphql/tag.graphql +++ b/datahub-web-react/src/graphql/tag.graphql @@ -30,3 +30,36 @@ query getTag($urn: String!) { } } } + +mutation updateTag($input: TagUpdate!) { + updateTag(input: $input) { + urn + name + description + ownership { + owners { + owner { + urn + type + username + info { + active + displayName + title + email + firstName + lastName + fullName + } + editableInfo { + pictureLink + } + } + type + } + lastModified { + time + } + } + } +} diff --git a/docs/features.md b/docs/features.md index ec3987a82e..e02c8dee47 100644 --- a/docs/features.md +++ b/docs/features.md @@ -18,6 +18,11 @@ Our open sourcing [blog post](https://engineering.linkedin.com/blog/2020/open-so - **Social actions**: likes, follows, bookmarks [*coming soon*] - **Compliance management**: field level tag based compliance editing [*coming soon*] - **Top users**: frequent users of a dataset [*coming soon*] + +### Tags + - **Globally defined**: Tags provided a standardized set of labels that can be shared across all your entities + - **Supports entities and schemas**: Tags can be applied at the entity level or for datasets, attached to schema fields. + - **Searchable** Entities can be searched and filtered by tag ### Users - **Search**: full-text & advanced search, search ranking diff --git a/gms/api/src/main/pegasus/com/linkedin/dataset/Dataset.pdl b/gms/api/src/main/pegasus/com/linkedin/dataset/Dataset.pdl index 2f9aa7a1b6..1ee2da6211 100644 --- a/gms/api/src/main/pegasus/com/linkedin/dataset/Dataset.pdl +++ b/gms/api/src/main/pegasus/com/linkedin/dataset/Dataset.pdl @@ -8,6 +8,7 @@ import com.linkedin.common.Status import com.linkedin.common.Uri import com.linkedin.common.VersionTag import com.linkedin.schema.SchemaMetadata +import com.linkedin.schema.EditableSchemaMetadata import com.linkedin.common.GlobalTags /** @@ -102,6 +103,11 @@ record Dataset includes DatasetKey, ChangeAuditStamps, VersionTag { */ schemaMetadata: optional SchemaMetadata + /** + * Editable schema metadata of the dataset + */ + editableSchemaMetadata: optional EditableSchemaMetadata + /** * Status metadata of the dataset */ @@ -116,4 +122,4 @@ record Dataset includes DatasetKey, ChangeAuditStamps, VersionTag { * List of global tags applied to the dataset */ globalTags: optional GlobalTags -} \ No newline at end of file +} diff --git a/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json index 604b07c06b..9e489c4190 100644 --- a/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json @@ -867,6 +867,44 @@ }, "doc" : "Schema metadata of the dataset", "optional" : true + }, { + "name" : "editableSchemaMetadata", + "type" : { + "type" : "record", + "name" : "EditableSchemaMetadata", + "namespace" : "com.linkedin.schema", + "doc" : "EditableSchemaMetadata stores editable changes made to schema metadata. This separates changes made from\ningestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines.", + "include" : [ "com.linkedin.common.ChangeAuditStamps" ], + "fields" : [ { + "name" : "editableSchemaFieldInfo", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "EditableSchemaFieldInfo", + "doc" : "SchemaField to describe metadata related to dataset schema.", + "fields" : [ { + "name" : "fieldPath", + "type" : "string", + "doc" : "FieldPath uniquely identifying the SchemaField this metadata is associated with" + }, { + "name" : "description", + "type" : "string", + "doc" : "Description", + "optional" : true + }, { + "name" : "globalTags", + "type" : "com.linkedin.common.GlobalTags", + "doc" : "Tags associated with the field", + "optional" : true + } ] + } + }, + "doc" : "Client provided a list of fields from document schema." + } ] + }, + "doc" : "Editable schema metadata of the dataset", + "optional" : true }, { "name" : "status", "type" : "com.linkedin.common.Status", @@ -1039,7 +1077,7 @@ "name" : "DatasetAspect", "namespace" : "com.linkedin.metadata.aspect", "doc" : "A union of all supported metadata aspects for a Dataset", - "ref" : [ "com.linkedin.dataset.DatasetProperties", "com.linkedin.dataset.DatasetDeprecation", "com.linkedin.dataset.DatasetUpstreamLineage", "com.linkedin.dataset.UpstreamLineage", "com.linkedin.common.InstitutionalMemory", "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.schema.SchemaMetadata", "com.linkedin.common.GlobalTags" ] + "ref" : [ "com.linkedin.dataset.DatasetProperties", "com.linkedin.dataset.DatasetDeprecation", "com.linkedin.dataset.DatasetUpstreamLineage", "com.linkedin.dataset.UpstreamLineage", "com.linkedin.common.InstitutionalMemory", "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.schema.SchemaMetadata", "com.linkedin.schema.EditableSchemaMetadata", "com.linkedin.common.GlobalTags" ] }, { "type" : "record", "name" : "AggregationMetadata", @@ -1345,7 +1383,7 @@ "validate" : { "com.linkedin.restli.common.EmptyRecordValidator" : { } } - }, "com.linkedin.schema.ArrayType", "com.linkedin.schema.BinaryJsonSchema", "com.linkedin.schema.BooleanType", "com.linkedin.schema.BytesType", "com.linkedin.schema.DatasetFieldForeignKey", "com.linkedin.schema.DateType", "com.linkedin.schema.EnumType", "com.linkedin.schema.EspressoSchema", "com.linkedin.schema.FixedType", "com.linkedin.schema.ForeignKeySpec", "com.linkedin.schema.KafkaSchema", "com.linkedin.schema.KeyValueSchema", "com.linkedin.schema.MapType", "com.linkedin.schema.MySqlDDL", "com.linkedin.schema.NullType", "com.linkedin.schema.NumberType", "com.linkedin.schema.OracleDDL", "com.linkedin.schema.OrcSchema", "com.linkedin.schema.OtherSchema", "com.linkedin.schema.PrestoDDL", "com.linkedin.schema.RecordType", "com.linkedin.schema.SchemaField", "com.linkedin.schema.SchemaFieldDataType", "com.linkedin.schema.SchemaMetadata", "com.linkedin.schema.SchemaMetadataKey", "com.linkedin.schema.Schemaless", "com.linkedin.schema.StringType", "com.linkedin.schema.TimeType", "com.linkedin.schema.UnionType", "com.linkedin.schema.UrnForeignKey" ], + }, "com.linkedin.schema.ArrayType", "com.linkedin.schema.BinaryJsonSchema", "com.linkedin.schema.BooleanType", "com.linkedin.schema.BytesType", "com.linkedin.schema.DatasetFieldForeignKey", "com.linkedin.schema.DateType", "com.linkedin.schema.EditableSchemaFieldInfo", "com.linkedin.schema.EditableSchemaMetadata", "com.linkedin.schema.EnumType", "com.linkedin.schema.EspressoSchema", "com.linkedin.schema.FixedType", "com.linkedin.schema.ForeignKeySpec", "com.linkedin.schema.KafkaSchema", "com.linkedin.schema.KeyValueSchema", "com.linkedin.schema.MapType", "com.linkedin.schema.MySqlDDL", "com.linkedin.schema.NullType", "com.linkedin.schema.NumberType", "com.linkedin.schema.OracleDDL", "com.linkedin.schema.OrcSchema", "com.linkedin.schema.OtherSchema", "com.linkedin.schema.PrestoDDL", "com.linkedin.schema.RecordType", "com.linkedin.schema.SchemaField", "com.linkedin.schema.SchemaFieldDataType", "com.linkedin.schema.SchemaMetadata", "com.linkedin.schema.SchemaMetadataKey", "com.linkedin.schema.Schemaless", "com.linkedin.schema.StringType", "com.linkedin.schema.TimeType", "com.linkedin.schema.UnionType", "com.linkedin.schema.UrnForeignKey" ], "schema" : { "name" : "datasets", "namespace" : "com.linkedin.dataset", diff --git a/gms/client/src/main/java/com/linkedin/chart/client/Charts.java b/gms/client/src/main/java/com/linkedin/chart/client/Charts.java index 1ffc4c9f1b..68b17ddab5 100644 --- a/gms/client/src/main/java/com/linkedin/chart/client/Charts.java +++ b/gms/client/src/main/java/com/linkedin/chart/client/Charts.java @@ -9,18 +9,26 @@ import com.linkedin.common.urn.ChartUrn; import com.linkedin.dashboard.Chart; import com.linkedin.dashboard.ChartKey; import com.linkedin.data.template.StringArray; +import com.linkedin.metadata.aspect.ChartAspect; import com.linkedin.metadata.configs.ChartSearchConfig; +import com.linkedin.metadata.dao.ChartActionRequestBuilder; +import com.linkedin.metadata.dao.utils.ModelUtils; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SortCriterion; import com.linkedin.metadata.restli.BaseBrowsableClient; +import com.linkedin.metadata.snapshot.ChartSnapshot; import com.linkedin.r2.RemoteInvocationException; import com.linkedin.restli.client.BatchGetEntityRequest; import com.linkedin.restli.client.Client; import com.linkedin.restli.client.GetRequest; +import com.linkedin.restli.client.Request; import com.linkedin.restli.common.CollectionResponse; import com.linkedin.restli.common.ComplexResourceKey; import com.linkedin.restli.common.EmptyRecord; + +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -33,6 +41,7 @@ import static com.linkedin.metadata.dao.utils.QueryUtils.newFilter; public class Charts extends BaseBrowsableClient { private static final ChartsRequestBuilders CHARTS_REQUEST_BUILDERS = new ChartsRequestBuilders(); + private static final ChartActionRequestBuilder CHARTS_ACTION_REQUEST_BUILDERS = new ChartActionRequestBuilder(); private static final ChartSearchConfig CHARTS_SEARCH_CONFIG = new ChartSearchConfig(); public Charts(@Nonnull Client restliClient) { @@ -144,6 +153,34 @@ public class Charts extends BaseBrowsableClient { return _client.sendRequest(requestBuilder.build()).getResponse().getEntity(); } + /** + * Update an existing Chart + */ + public void update(@Nonnull final ChartUrn urn, @Nonnull final Chart chart) throws RemoteInvocationException { + Request request = CHARTS_ACTION_REQUEST_BUILDERS.createRequest(urn, toSnapshot(chart, urn)); + _client.sendRequest(request).getResponse(); + } + + static ChartSnapshot toSnapshot(@Nonnull Chart chart, @Nonnull ChartUrn urn) { + final List aspects = new ArrayList<>(); + if (chart.hasInfo()) { + aspects.add(ModelUtils.newAspectUnion(ChartAspect.class, chart.getInfo())); + } + if (chart.hasQuery()) { + aspects.add(ModelUtils.newAspectUnion(ChartAspect.class, chart.getQuery())); + } + if (chart.hasOwnership()) { + aspects.add(ModelUtils.newAspectUnion(ChartAspect.class, chart.getOwnership())); + } + if (chart.hasStatus()) { + aspects.add(ModelUtils.newAspectUnion(ChartAspect.class, chart.getStatus())); + } + if (chart.hasGlobalTags()) { + aspects.add(ModelUtils.newAspectUnion(ChartAspect.class, chart.getGlobalTags())); + } + return ModelUtils.newSnapshot(ChartSnapshot.class, urn, aspects); + } + @Nonnull private ComplexResourceKey getKeyFromUrn(@Nonnull ChartUrn urn) { return new ComplexResourceKey<>(toChartKey(urn), new EmptyRecord()); diff --git a/gms/client/src/main/java/com/linkedin/dashboard/client/Dashboards.java b/gms/client/src/main/java/com/linkedin/dashboard/client/Dashboards.java index 8b468e49e3..3a4b9cd83b 100644 --- a/gms/client/src/main/java/com/linkedin/dashboard/client/Dashboards.java +++ b/gms/client/src/main/java/com/linkedin/dashboard/client/Dashboards.java @@ -9,18 +9,26 @@ import com.linkedin.dashboard.DashboardsDoGetBrowsePathsRequestBuilder; import com.linkedin.dashboard.DashboardsFindBySearchRequestBuilder; import com.linkedin.dashboard.DashboardsRequestBuilders; import com.linkedin.data.template.StringArray; +import com.linkedin.metadata.aspect.DashboardAspect; import com.linkedin.metadata.configs.DashboardSearchConfig; +import com.linkedin.metadata.dao.DashboardActionRequestBuilder; +import com.linkedin.metadata.dao.utils.ModelUtils; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SortCriterion; import com.linkedin.metadata.restli.BaseBrowsableClient; +import com.linkedin.metadata.snapshot.DashboardSnapshot; import com.linkedin.r2.RemoteInvocationException; import com.linkedin.restli.client.BatchGetEntityRequest; import com.linkedin.restli.client.Client; import com.linkedin.restli.client.GetRequest; +import com.linkedin.restli.client.Request; import com.linkedin.restli.common.CollectionResponse; import com.linkedin.restli.common.ComplexResourceKey; import com.linkedin.restli.common.EmptyRecord; + +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -33,6 +41,7 @@ import static com.linkedin.metadata.dao.utils.QueryUtils.newFilter; public class Dashboards extends BaseBrowsableClient { private static final DashboardsRequestBuilders DASHBOARDS_REQUEST_BUILDERS = new DashboardsRequestBuilders(); + private static final DashboardActionRequestBuilder DASHBOARDS_ACTION_REQUEST_BUILDERS = new DashboardActionRequestBuilder(); private static final DashboardSearchConfig DASHBOARDS_SEARCH_CONFIG = new DashboardSearchConfig(); public Dashboards(@Nonnull Client restliClient) { @@ -144,6 +153,31 @@ public class Dashboards extends BaseBrowsableClient { return _client.sendRequest(requestBuilder.build()).getResponse().getEntity(); } + /** + * Update an existing Dashboard + */ + public void update(@Nonnull final DashboardUrn urn, @Nonnull final Dashboard dashboard) throws RemoteInvocationException { + Request request = DASHBOARDS_ACTION_REQUEST_BUILDERS.createRequest(urn, toSnapshot(dashboard, urn)); + _client.sendRequest(request).getResponse(); + } + + static DashboardSnapshot toSnapshot(@Nonnull Dashboard dashboard, @Nonnull DashboardUrn urn) { + final List aspects = new ArrayList<>(); + if (dashboard.hasInfo()) { + aspects.add(ModelUtils.newAspectUnion(DashboardAspect.class, dashboard.getInfo())); + } + if (dashboard.hasOwnership()) { + aspects.add(ModelUtils.newAspectUnion(DashboardAspect.class, dashboard.getOwnership())); + } + if (dashboard.hasStatus()) { + aspects.add(ModelUtils.newAspectUnion(DashboardAspect.class, dashboard.getStatus())); + } + if (dashboard.hasGlobalTags()) { + aspects.add(ModelUtils.newAspectUnion(DashboardAspect.class, dashboard.getGlobalTags())); + } + return ModelUtils.newSnapshot(DashboardSnapshot.class, urn, aspects); + } + @Nonnull private ComplexResourceKey getKeyFromUrn(@Nonnull DashboardUrn urn) { return new ComplexResourceKey<>(toDashboardsKey(urn), new EmptyRecord()); diff --git a/gms/client/src/main/java/com/linkedin/dataset/client/Datasets.java b/gms/client/src/main/java/com/linkedin/dataset/client/Datasets.java index 86ba617d59..6c1a73cb91 100644 --- a/gms/client/src/main/java/com/linkedin/dataset/client/Datasets.java +++ b/gms/client/src/main/java/com/linkedin/dataset/client/Datasets.java @@ -309,6 +309,12 @@ public class Datasets extends BaseBrowsableClient { if (dataset.hasRemoved()) { aspects.add(DatasetAspect.create(new Status().setRemoved(dataset.isRemoved()))); } + if (dataset.getGlobalTags() != null) { + aspects.add(ModelUtils.newAspectUnion(DatasetAspect.class, dataset.getGlobalTags())); + } + if (dataset.getEditableSchemaMetadata() != null) { + aspects.add(ModelUtils.newAspectUnion(DatasetAspect.class, dataset.getEditableSchemaMetadata())); + } return ModelUtils.newSnapshot(DatasetSnapshot.class, datasetUrn, aspects); } diff --git a/gms/client/src/main/java/com/linkedin/tag/client/Tags.java b/gms/client/src/main/java/com/linkedin/tag/client/Tags.java index 76fdf7491f..ff34a0887f 100644 --- a/gms/client/src/main/java/com/linkedin/tag/client/Tags.java +++ b/gms/client/src/main/java/com/linkedin/tag/client/Tags.java @@ -2,24 +2,31 @@ package com.linkedin.tag.client; import com.linkedin.common.urn.TagUrn; import com.linkedin.data.template.StringArray; +import com.linkedin.metadata.aspect.TagAspect; import com.linkedin.metadata.configs.TagSearchConfig; +import com.linkedin.metadata.dao.TagActionRequestBuilders; +import com.linkedin.metadata.dao.utils.ModelUtils; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.SortCriterion; import com.linkedin.metadata.restli.BaseSearchableClient; +import com.linkedin.metadata.snapshot.TagSnapshot; import com.linkedin.r2.RemoteInvocationException; import com.linkedin.restli.client.BatchGetEntityRequest; import com.linkedin.restli.client.Client; import com.linkedin.restli.client.GetAllRequest; import com.linkedin.restli.client.GetRequest; +import com.linkedin.restli.client.Request; import com.linkedin.restli.common.CollectionResponse; import com.linkedin.restli.common.ComplexResourceKey; import com.linkedin.restli.common.EmptyRecord; import com.linkedin.tag.Tag; import com.linkedin.tag.TagKey; +import com.linkedin.tag.TagProperties; import com.linkedin.tag.TagsDoAutocompleteRequestBuilder; import com.linkedin.tag.TagsFindBySearchRequestBuilder; import com.linkedin.tag.TagsRequestBuilders; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -32,6 +39,7 @@ import static com.linkedin.metadata.dao.utils.QueryUtils.newFilter; public class Tags extends BaseSearchableClient { private static final TagsRequestBuilders TAGS_REQUEST_BUILDERS = new TagsRequestBuilders(); + private static final TagActionRequestBuilders TAGS_ACTION_REQUEST_BUILDERS = new TagActionRequestBuilders(); private static final TagSearchConfig TAGS_SEARCH_CONFIG = new TagSearchConfig(); public Tags(@Nonnull Client restliClient) { @@ -119,6 +127,28 @@ public class Tags extends BaseSearchableClient { return _client.sendRequest(requestBuilder.build()).getResponse().getEntity(); } + /** + * Update an existing Tag + */ + public void update(@Nonnull final TagUrn urn, @Nonnull final Tag tag) throws RemoteInvocationException { + Request request = TAGS_ACTION_REQUEST_BUILDERS.createRequest(urn, toSnapshot(tag, urn)); + _client.sendRequest(request).getResponse(); + } + + static TagSnapshot toSnapshot(@Nonnull Tag tag, @Nonnull TagUrn tagUrn) { + final List aspects = new ArrayList<>(); + if (tag.hasDescription()) { + TagProperties tagProperties = new TagProperties(); + tagProperties.setDescription((tag.getDescription())); + tagProperties.setName((tag.getName())); + aspects.add(ModelUtils.newAspectUnion(TagAspect.class, tagProperties)); + } + if (tag.hasOwnership()) { + aspects.add(ModelUtils.newAspectUnion(TagAspect.class, tag.getOwnership())); + } + return ModelUtils.newSnapshot(TagSnapshot.class, tagUrn, aspects); + } + /** * Get all {@link Tag} models of the tag * diff --git a/gms/impl/src/main/java/com/linkedin/metadata/resources/dataset/Datasets.java b/gms/impl/src/main/java/com/linkedin/metadata/resources/dataset/Datasets.java index 73efce83ae..7b3b5afb3f 100644 --- a/gms/impl/src/main/java/com/linkedin/metadata/resources/dataset/Datasets.java +++ b/gms/impl/src/main/java/com/linkedin/metadata/resources/dataset/Datasets.java @@ -40,6 +40,7 @@ import com.linkedin.restli.server.annotations.PagingContextParam; import com.linkedin.restli.server.annotations.QueryParam; import com.linkedin.restli.server.annotations.RestLiCollection; import com.linkedin.restli.server.annotations.RestMethod; +import com.linkedin.schema.EditableSchemaMetadata; import com.linkedin.schema.SchemaMetadata; import java.util.ArrayList; import java.util.List; @@ -155,8 +156,10 @@ public final class Datasets extends BaseBrowsableEntityResource< value.setUpstreamLineage((UpstreamLineage) aspect); } else if (aspect instanceof GlobalTags) { value.setGlobalTags(GlobalTags.class.cast(aspect)); + } else if (aspect instanceof EditableSchemaMetadata) { + value.setEditableSchemaMetadata(EditableSchemaMetadata.class.cast(aspect)); } - }); + }); return value; } @@ -191,6 +194,9 @@ public final class Datasets extends BaseBrowsableEntityResource< if (dataset.hasGlobalTags()) { aspects.add(ModelUtils.newAspectUnion(DatasetAspect.class, dataset.getGlobalTags())); } + if (dataset.hasEditableSchemaMetadata()) { + aspects.add(ModelUtils.newAspectUnion(DatasetAspect.class, dataset.getEditableSchemaMetadata())); + } return ModelUtils.newSnapshot(DatasetSnapshot.class, datasetUrn, aspects); } diff --git a/metadata-dao-impl/restli-dao/src/main/java/com/linkedin/metadata/dao/TagActionRequestBuilders.java b/metadata-dao-impl/restli-dao/src/main/java/com/linkedin/metadata/dao/TagActionRequestBuilders.java new file mode 100644 index 0000000000..bf70dfc546 --- /dev/null +++ b/metadata-dao-impl/restli-dao/src/main/java/com/linkedin/metadata/dao/TagActionRequestBuilders.java @@ -0,0 +1,16 @@ +package com.linkedin.metadata.dao; + +import com.linkedin.common.urn.TagUrn; +import com.linkedin.metadata.snapshot.TagSnapshot; + +/** + * An action request builder for tag entities. + */ +public class TagActionRequestBuilders extends BaseActionRequestBuilder { + + private static final String BASE_URI_TEMPLATE = "tags"; + + public TagActionRequestBuilders() { + super(TagSnapshot.class, TagUrn.class, BASE_URI_TEMPLATE); + } +} diff --git a/metadata-ingestion/examples/mce_files/bootstrap_mce.json b/metadata-ingestion/examples/mce_files/bootstrap_mce.json index 6d20ba6a9b..ea5df16c8a 100644 --- a/metadata-ingestion/examples/mce_files/bootstrap_mce.json +++ b/metadata-ingestion/examples/mce_files/bootstrap_mce.json @@ -118,6 +118,27 @@ ] } }, + { + "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata": { + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "editableSchemaFieldInfo": [ + { + "fieldPath": "field_foo", + "globalTags": { "tags": [{ "tag": "urn:li:tag:Legacy" }] } + } + ] + } + }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { "schemaName": "SampleKafkaSchema", @@ -156,6 +177,7 @@ } }, "nativeDataType": "string", + "globalTags": { "tags": [{ "tag": "urn:li:tag:NeedsDocumentation" }] }, "recursive": false }, { @@ -475,7 +497,7 @@ }, "proposedDelta": null }, - { + { "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -756,5 +778,38 @@ } }, "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:NeedsDocumentation", + "aspects": [ + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "NeedsDocumentation", + "description": "Indicates the data element needs documentation" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + "proposedDelta": null } ] diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/DatasetAspect.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/DatasetAspect.pdl index 100c79f327..ecaaca637c 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/DatasetAspect.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/DatasetAspect.pdl @@ -8,6 +8,7 @@ import com.linkedin.dataset.DatasetProperties import com.linkedin.dataset.DatasetUpstreamLineage import com.linkedin.dataset.UpstreamLineage import com.linkedin.schema.SchemaMetadata +import com.linkedin.schema.EditableSchemaMetadata import com.linkedin.common.GlobalTags /** @@ -22,5 +23,6 @@ typeref DatasetAspect = union[ Ownership, Status, SchemaMetadata, + EditableSchemaMetadata, GlobalTags -] \ No newline at end of file +] diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl new file mode 100644 index 0000000000..0909cbaee8 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl @@ -0,0 +1,23 @@ +namespace com.linkedin.schema + +import com.linkedin.common.GlobalTags + +/** + * SchemaField to describe metadata related to dataset schema. + */ +record EditableSchemaFieldInfo { + /** + * FieldPath uniquely identifying the SchemaField this metadata is associated with + */ + fieldPath: string + + /** + * Description + */ + description: optional string + + /** + * Tags associated with the field + */ + globalTags: optional GlobalTags +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaMetadata.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaMetadata.pdl new file mode 100644 index 0000000000..15429cc791 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaMetadata.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.schema + +import com.linkedin.common.ChangeAuditStamps + +/** + * EditableSchemaMetadata stores editable changes made to schema metadata. This separates changes made from + * ingestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines. + */ +record EditableSchemaMetadata includes ChangeAuditStamps { + /** + * Client provided a list of fields from document schema. + */ + editableSchemaFieldInfo: array[EditableSchemaFieldInfo] +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl index 0b636e5876..f417d19a6e 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl @@ -47,4 +47,4 @@ record SchemaField { * Tags associated with the field */ globalTags: optional GlobalTags -} \ No newline at end of file +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaMetadata.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaMetadata.pdl index 0a888b3eef..0df7a570ba 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaMetadata.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaMetadata.pdl @@ -54,4 +54,4 @@ record SchemaMetadata includes SchemaMetadataKey, ChangeAuditStamps { * Map captures all the references schema makes to external datasets. Map key is ForeignKeySpecName typeref. */ foreignKeysSpecs: optional map[string, ForeignKeySpec] -} \ No newline at end of file +}