mirror of
https://github.com/datahub-project/datahub.git
synced 2025-10-10 00:13:49 +00:00
feat(structuredProperties): add hide property and show as badge validators (#12099)
Co-authored-by: RyanHolstien <RyanHolstien@users.noreply.github.com>
This commit is contained in:
parent
7f846dcbdb
commit
d2eaf0c83c
@ -0,0 +1,35 @@
|
|||||||
|
package com.linkedin.metadata.entity;
|
||||||
|
|
||||||
|
import com.linkedin.metadata.query.filter.Filter;
|
||||||
|
import com.linkedin.metadata.search.ScrollResult;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
import lombok.Builder;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetches pages of structured properties which have been applied to an entity urn with a specified
|
||||||
|
* filter
|
||||||
|
*/
|
||||||
|
@Builder
|
||||||
|
public class GenericScrollIterator implements Iterator<ScrollResult> {
|
||||||
|
@Nonnull private final Filter filter;
|
||||||
|
@Nonnull private final List<String> entities;
|
||||||
|
@Nonnull private final SearchRetriever searchRetriever;
|
||||||
|
private int count;
|
||||||
|
@Builder.Default private String scrollId = null;
|
||||||
|
@Builder.Default private boolean started = false;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasNext() {
|
||||||
|
return !started || scrollId != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ScrollResult next() {
|
||||||
|
started = true;
|
||||||
|
ScrollResult result = searchRetriever.scroll(entities, filter, scrollId, count);
|
||||||
|
scrollId = result.getScrollId();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
@ -3,8 +3,6 @@ package com.linkedin.metadata.structuredproperties.hooks;
|
|||||||
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME;
|
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME;
|
||||||
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
|
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
|
||||||
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_KEY_ASPECT_NAME;
|
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_KEY_ASPECT_NAME;
|
||||||
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX;
|
|
||||||
import static com.linkedin.metadata.utils.CriterionUtils.buildExistsCriterion;
|
|
||||||
|
|
||||||
import com.linkedin.common.AuditStamp;
|
import com.linkedin.common.AuditStamp;
|
||||||
import com.linkedin.common.urn.Urn;
|
import com.linkedin.common.urn.Urn;
|
||||||
@ -17,30 +15,19 @@ import com.linkedin.metadata.aspect.patch.GenericJsonPatch;
|
|||||||
import com.linkedin.metadata.aspect.patch.PatchOperationType;
|
import com.linkedin.metadata.aspect.patch.PatchOperationType;
|
||||||
import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
|
import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
|
||||||
import com.linkedin.metadata.aspect.plugins.hooks.MCPSideEffect;
|
import com.linkedin.metadata.aspect.plugins.hooks.MCPSideEffect;
|
||||||
import com.linkedin.metadata.entity.SearchRetriever;
|
|
||||||
import com.linkedin.metadata.entity.ebean.batch.PatchItemImpl;
|
import com.linkedin.metadata.entity.ebean.batch.PatchItemImpl;
|
||||||
import com.linkedin.metadata.models.EntitySpec;
|
import com.linkedin.metadata.models.EntitySpec;
|
||||||
import com.linkedin.metadata.models.StructuredPropertyUtils;
|
import com.linkedin.metadata.structuredproperties.util.EntityWithPropertyIterator;
|
||||||
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
|
|
||||||
import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
|
|
||||||
import com.linkedin.metadata.query.filter.Criterion;
|
|
||||||
import com.linkedin.metadata.query.filter.CriterionArray;
|
|
||||||
import com.linkedin.metadata.query.filter.Filter;
|
|
||||||
import com.linkedin.metadata.search.ScrollResult;
|
|
||||||
import com.linkedin.structured.StructuredPropertyDefinition;
|
import com.linkedin.structured.StructuredPropertyDefinition;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.Iterator;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Spliterator;
|
import java.util.Spliterator;
|
||||||
import java.util.Spliterators;
|
import java.util.Spliterators;
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
import java.util.stream.StreamSupport;
|
import java.util.stream.StreamSupport;
|
||||||
import javax.annotation.Nonnull;
|
import javax.annotation.Nonnull;
|
||||||
import javax.annotation.Nullable;
|
import javax.annotation.Nullable;
|
||||||
import lombok.Builder;
|
|
||||||
import lombok.Getter;
|
import lombok.Getter;
|
||||||
import lombok.Setter;
|
import lombok.Setter;
|
||||||
import lombok.experimental.Accessors;
|
import lombok.experimental.Accessors;
|
||||||
@ -141,60 +128,4 @@ public class PropertyDefinitionDeleteSideEffect extends MCPSideEffect {
|
|||||||
.build(retrieverContext.getAspectRetriever().getEntityRegistry());
|
.build(retrieverContext.getAspectRetriever().getEntityRegistry());
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Fetches pages of entity urns which have a value for the given structured property definition
|
|
||||||
*/
|
|
||||||
@Builder
|
|
||||||
public static class EntityWithPropertyIterator implements Iterator<ScrollResult> {
|
|
||||||
@Nonnull private final Urn propertyUrn;
|
|
||||||
@Nullable private final StructuredPropertyDefinition definition;
|
|
||||||
@Nonnull private final SearchRetriever searchRetriever;
|
|
||||||
private int count;
|
|
||||||
@Builder.Default private String scrollId = null;
|
|
||||||
@Builder.Default private boolean started = false;
|
|
||||||
|
|
||||||
private List<String> getEntities() {
|
|
||||||
if (definition != null && definition.getEntityTypes() != null) {
|
|
||||||
return definition.getEntityTypes().stream()
|
|
||||||
.map(StructuredPropertyUtils::getValueTypeId)
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
} else {
|
|
||||||
return Collections.emptyList();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Filter getFilter() {
|
|
||||||
Filter propertyFilter = new Filter();
|
|
||||||
final ConjunctiveCriterionArray disjunction = new ConjunctiveCriterionArray();
|
|
||||||
final ConjunctiveCriterion conjunction = new ConjunctiveCriterion();
|
|
||||||
final CriterionArray andCriterion = new CriterionArray();
|
|
||||||
|
|
||||||
// Cannot rely on automatic field name since the definition is deleted
|
|
||||||
final Criterion propertyExistsCriterion =
|
|
||||||
buildExistsCriterion(
|
|
||||||
STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX
|
|
||||||
+ StructuredPropertyUtils.toElasticsearchFieldName(propertyUrn, definition));
|
|
||||||
|
|
||||||
andCriterion.add(propertyExistsCriterion);
|
|
||||||
conjunction.setAnd(andCriterion);
|
|
||||||
disjunction.add(conjunction);
|
|
||||||
propertyFilter.setOr(disjunction);
|
|
||||||
|
|
||||||
return propertyFilter;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean hasNext() {
|
|
||||||
return !started || scrollId != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ScrollResult next() {
|
|
||||||
started = true;
|
|
||||||
ScrollResult result = searchRetriever.scroll(getEntities(), getFilter(), scrollId, count);
|
|
||||||
scrollId = result.getScrollId();
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,76 @@
|
|||||||
|
package com.linkedin.metadata.structuredproperties.util;
|
||||||
|
|
||||||
|
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX;
|
||||||
|
import static com.linkedin.metadata.utils.CriterionUtils.buildExistsCriterion;
|
||||||
|
|
||||||
|
import com.linkedin.common.urn.Urn;
|
||||||
|
import com.linkedin.metadata.entity.SearchRetriever;
|
||||||
|
import com.linkedin.metadata.models.StructuredPropertyUtils;
|
||||||
|
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
|
||||||
|
import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
|
||||||
|
import com.linkedin.metadata.query.filter.Criterion;
|
||||||
|
import com.linkedin.metadata.query.filter.CriterionArray;
|
||||||
|
import com.linkedin.metadata.query.filter.Filter;
|
||||||
|
import com.linkedin.metadata.search.ScrollResult;
|
||||||
|
import com.linkedin.structured.StructuredPropertyDefinition;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
import javax.annotation.Nullable;
|
||||||
|
import lombok.Builder;
|
||||||
|
|
||||||
|
/** Fetches pages of entity urns which have a value for the given structured property definition */
|
||||||
|
@Builder
|
||||||
|
public class EntityWithPropertyIterator implements Iterator<ScrollResult> {
|
||||||
|
@Nonnull private final Urn propertyUrn;
|
||||||
|
@Nullable private final StructuredPropertyDefinition definition;
|
||||||
|
@Nonnull private final SearchRetriever searchRetriever;
|
||||||
|
private int count;
|
||||||
|
@Builder.Default private String scrollId = null;
|
||||||
|
@Builder.Default private boolean started = false;
|
||||||
|
|
||||||
|
private List<String> getEntities() {
|
||||||
|
if (definition != null && definition.getEntityTypes() != null) {
|
||||||
|
return definition.getEntityTypes().stream()
|
||||||
|
.map(StructuredPropertyUtils::getValueTypeId)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
} else {
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Filter getFilter() {
|
||||||
|
Filter propertyFilter = new Filter();
|
||||||
|
final ConjunctiveCriterionArray disjunction = new ConjunctiveCriterionArray();
|
||||||
|
final ConjunctiveCriterion conjunction = new ConjunctiveCriterion();
|
||||||
|
final CriterionArray andCriterion = new CriterionArray();
|
||||||
|
|
||||||
|
// Cannot rely on automatic field name since the definition is deleted
|
||||||
|
final Criterion propertyExistsCriterion =
|
||||||
|
buildExistsCriterion(
|
||||||
|
STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX
|
||||||
|
+ StructuredPropertyUtils.toElasticsearchFieldName(propertyUrn, definition));
|
||||||
|
|
||||||
|
andCriterion.add(propertyExistsCriterion);
|
||||||
|
conjunction.setAnd(andCriterion);
|
||||||
|
disjunction.add(conjunction);
|
||||||
|
propertyFilter.setOr(disjunction);
|
||||||
|
|
||||||
|
return propertyFilter;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasNext() {
|
||||||
|
return !started || scrollId != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ScrollResult next() {
|
||||||
|
started = true;
|
||||||
|
ScrollResult result = searchRetriever.scroll(getEntities(), getFilter(), scrollId, count);
|
||||||
|
scrollId = result.getScrollId();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,63 @@
|
|||||||
|
package com.linkedin.metadata.structuredproperties.validation;
|
||||||
|
|
||||||
|
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
import com.linkedin.metadata.aspect.RetrieverContext;
|
||||||
|
import com.linkedin.metadata.aspect.batch.BatchItem;
|
||||||
|
import com.linkedin.metadata.aspect.batch.ChangeMCP;
|
||||||
|
import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
|
||||||
|
import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator;
|
||||||
|
import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
|
||||||
|
import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection;
|
||||||
|
import com.linkedin.metadata.models.StructuredPropertyUtils;
|
||||||
|
import com.linkedin.structured.StructuredPropertySettings;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import java.util.stream.Stream;
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
import lombok.experimental.Accessors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@Setter
|
||||||
|
@Getter
|
||||||
|
@Slf4j
|
||||||
|
@Accessors(chain = true)
|
||||||
|
public class HidePropertyValidator extends AspectPayloadValidator {
|
||||||
|
|
||||||
|
@Nonnull private AspectPluginConfig config;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Stream<AspectValidationException> validateProposedAspects(
|
||||||
|
@Nonnull Collection<? extends BatchItem> mcpItems,
|
||||||
|
@Nonnull RetrieverContext retrieverContext) {
|
||||||
|
return validateSettingsUpserts(
|
||||||
|
mcpItems.stream()
|
||||||
|
.filter(i -> STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME.equals(i.getAspectName()))
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Stream<AspectValidationException> validatePreCommitAspects(
|
||||||
|
@Nonnull Collection<ChangeMCP> changeMCPs, @Nonnull RetrieverContext retrieverContext) {
|
||||||
|
return Stream.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
public static Stream<AspectValidationException> validateSettingsUpserts(
|
||||||
|
@Nonnull Collection<? extends BatchItem> mcpItems) {
|
||||||
|
ValidationExceptionCollection exceptions = ValidationExceptionCollection.newCollection();
|
||||||
|
for (BatchItem mcpItem : mcpItems) {
|
||||||
|
StructuredPropertySettings structuredPropertySettings =
|
||||||
|
mcpItem.getAspect(StructuredPropertySettings.class);
|
||||||
|
boolean isValid =
|
||||||
|
StructuredPropertyUtils.validatePropertySettings(structuredPropertySettings, false);
|
||||||
|
if (!isValid) {
|
||||||
|
exceptions.addException(mcpItem, StructuredPropertyUtils.INVALID_SETTINGS_MESSAGE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return exceptions.streamAllExceptions();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,144 @@
|
|||||||
|
package com.linkedin.metadata.structuredproperties.validation;
|
||||||
|
|
||||||
|
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME;
|
||||||
|
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME;
|
||||||
|
import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion;
|
||||||
|
|
||||||
|
import com.datahub.util.RecordUtils;
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
import com.linkedin.entity.Aspect;
|
||||||
|
import com.linkedin.metadata.aspect.AspectRetriever;
|
||||||
|
import com.linkedin.metadata.aspect.RetrieverContext;
|
||||||
|
import com.linkedin.metadata.aspect.batch.BatchItem;
|
||||||
|
import com.linkedin.metadata.aspect.batch.ChangeMCP;
|
||||||
|
import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
|
||||||
|
import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator;
|
||||||
|
import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
|
||||||
|
import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection;
|
||||||
|
import com.linkedin.metadata.entity.GenericScrollIterator;
|
||||||
|
import com.linkedin.metadata.models.StructuredPropertyUtils;
|
||||||
|
import com.linkedin.metadata.query.filter.Condition;
|
||||||
|
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
|
||||||
|
import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
|
||||||
|
import com.linkedin.metadata.query.filter.Criterion;
|
||||||
|
import com.linkedin.metadata.query.filter.CriterionArray;
|
||||||
|
import com.linkedin.metadata.query.filter.Filter;
|
||||||
|
import com.linkedin.metadata.search.ScrollResult;
|
||||||
|
import com.linkedin.metadata.search.SearchEntity;
|
||||||
|
import com.linkedin.structured.StructuredPropertySettings;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import java.util.stream.Stream;
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
import lombok.experimental.Accessors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
|
|
||||||
|
@Setter
|
||||||
|
@Getter
|
||||||
|
@Slf4j
|
||||||
|
@Accessors(chain = true)
|
||||||
|
public class ShowPropertyAsBadgeValidator extends AspectPayloadValidator {
|
||||||
|
|
||||||
|
@Nonnull private AspectPluginConfig config;
|
||||||
|
|
||||||
|
private static final String SHOW_ASSET_AS_BADGE_FIELD = "showAsAssetBadge";
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Stream<AspectValidationException> validateProposedAspects(
|
||||||
|
@Nonnull Collection<? extends BatchItem> mcpItems,
|
||||||
|
@Nonnull RetrieverContext retrieverContext) {
|
||||||
|
return validateSettingsUpserts(
|
||||||
|
mcpItems.stream()
|
||||||
|
.filter(i -> STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME.equals(i.getAspectName()))
|
||||||
|
.collect(Collectors.toList()),
|
||||||
|
retrieverContext);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Stream<AspectValidationException> validatePreCommitAspects(
|
||||||
|
@Nonnull Collection<ChangeMCP> changeMCPs, @Nonnull RetrieverContext retrieverContext) {
|
||||||
|
return Stream.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
public static Stream<AspectValidationException> validateSettingsUpserts(
|
||||||
|
@Nonnull Collection<? extends BatchItem> mcpItems,
|
||||||
|
@Nonnull RetrieverContext retrieverContext) {
|
||||||
|
ValidationExceptionCollection exceptions = ValidationExceptionCollection.newCollection();
|
||||||
|
for (BatchItem mcpItem : mcpItems) {
|
||||||
|
StructuredPropertySettings structuredPropertySettings =
|
||||||
|
mcpItem.getAspect(StructuredPropertySettings.class);
|
||||||
|
if (structuredPropertySettings.isShowAsAssetBadge()) {
|
||||||
|
// Search for any structured properties that have showAsAssetBadge set, should only ever be
|
||||||
|
// one at most.
|
||||||
|
GenericScrollIterator scrollIterator =
|
||||||
|
GenericScrollIterator.builder()
|
||||||
|
.searchRetriever(retrieverContext.getSearchRetriever())
|
||||||
|
.count(10) // Get first 10, should only ever be one, but this gives us more info if
|
||||||
|
// we're in a bad state
|
||||||
|
.filter(getFilter())
|
||||||
|
.entities(Collections.singletonList(STRUCTURED_PROPERTY_ENTITY_NAME))
|
||||||
|
.build();
|
||||||
|
// Only need to get first set, if there are more then will have to resolve bad state
|
||||||
|
ScrollResult scrollResult = scrollIterator.next();
|
||||||
|
if (CollectionUtils.isNotEmpty(scrollResult.getEntities())) {
|
||||||
|
if (scrollResult.getEntities().size() > 1) {
|
||||||
|
// If it's greater than one, don't bother querying DB since we for sure are in a bad
|
||||||
|
// state
|
||||||
|
exceptions.addException(
|
||||||
|
mcpItem,
|
||||||
|
StructuredPropertyUtils.ONLY_ONE_BADGE
|
||||||
|
+ scrollResult.getEntities().stream()
|
||||||
|
.map(SearchEntity::getEntity)
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
} else {
|
||||||
|
// If there is just one, verify against DB to make sure we're not hitting a timing issue
|
||||||
|
// with eventual consistency
|
||||||
|
AspectRetriever aspectRetriever = retrieverContext.getAspectRetriever();
|
||||||
|
Optional<Aspect> propertySettings =
|
||||||
|
Optional.ofNullable(
|
||||||
|
aspectRetriever.getLatestAspectObject(
|
||||||
|
scrollResult.getEntities().get(0).getEntity(),
|
||||||
|
STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME));
|
||||||
|
if (propertySettings.isPresent()) {
|
||||||
|
StructuredPropertySettings dbBadgeSettings =
|
||||||
|
RecordUtils.toRecordTemplate(
|
||||||
|
StructuredPropertySettings.class, propertySettings.get().data());
|
||||||
|
if (dbBadgeSettings.isShowAsAssetBadge()) {
|
||||||
|
exceptions.addException(
|
||||||
|
mcpItem,
|
||||||
|
StructuredPropertyUtils.ONLY_ONE_BADGE
|
||||||
|
+ scrollResult.getEntities().stream()
|
||||||
|
.map(SearchEntity::getEntity)
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return exceptions.streamAllExceptions();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Filter getFilter() {
|
||||||
|
Filter propertyFilter = new Filter();
|
||||||
|
final ConjunctiveCriterionArray disjunction = new ConjunctiveCriterionArray();
|
||||||
|
final ConjunctiveCriterion conjunction = new ConjunctiveCriterion();
|
||||||
|
final CriterionArray andCriterion = new CriterionArray();
|
||||||
|
|
||||||
|
final Criterion propertyExistsCriterion =
|
||||||
|
buildCriterion(SHOW_ASSET_AS_BADGE_FIELD, Condition.EQUAL, "true");
|
||||||
|
|
||||||
|
andCriterion.add(propertyExistsCriterion);
|
||||||
|
conjunction.setAnd(andCriterion);
|
||||||
|
disjunction.add(conjunction);
|
||||||
|
propertyFilter.setOr(disjunction);
|
||||||
|
|
||||||
|
return propertyFilter;
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,55 @@
|
|||||||
|
package com.linkedin.metadata.structuredproperties.validators;
|
||||||
|
|
||||||
|
import com.linkedin.common.urn.Urn;
|
||||||
|
import com.linkedin.common.urn.UrnUtils;
|
||||||
|
import com.linkedin.metadata.models.registry.EntityRegistry;
|
||||||
|
import com.linkedin.metadata.structuredproperties.validation.HidePropertyValidator;
|
||||||
|
import com.linkedin.structured.StructuredPropertySettings;
|
||||||
|
import com.linkedin.test.metadata.aspect.TestEntityRegistry;
|
||||||
|
import com.linkedin.test.metadata.aspect.batch.TestMCP;
|
||||||
|
import org.testng.Assert;
|
||||||
|
import org.testng.annotations.Test;
|
||||||
|
|
||||||
|
public class HidePropertyValidatorTest {
|
||||||
|
|
||||||
|
private static final EntityRegistry TEST_REGISTRY = new TestEntityRegistry();
|
||||||
|
|
||||||
|
private static final Urn TEST_PROPERTY_URN =
|
||||||
|
UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.privacy.retentionTime");
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testValidUpsert() {
|
||||||
|
|
||||||
|
StructuredPropertySettings propertySettings =
|
||||||
|
new StructuredPropertySettings()
|
||||||
|
.setIsHidden(false)
|
||||||
|
.setShowAsAssetBadge(true)
|
||||||
|
.setShowInAssetSummary(true)
|
||||||
|
.setShowInSearchFilters(true);
|
||||||
|
|
||||||
|
boolean isValid =
|
||||||
|
HidePropertyValidator.validateSettingsUpserts(
|
||||||
|
TestMCP.ofOneUpsertItem(TEST_PROPERTY_URN, propertySettings, TEST_REGISTRY))
|
||||||
|
.findAny()
|
||||||
|
.isEmpty();
|
||||||
|
Assert.assertTrue(isValid);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testInvalidUpsert() {
|
||||||
|
|
||||||
|
StructuredPropertySettings propertySettings =
|
||||||
|
new StructuredPropertySettings()
|
||||||
|
.setIsHidden(true)
|
||||||
|
.setShowAsAssetBadge(true)
|
||||||
|
.setShowInAssetSummary(true)
|
||||||
|
.setShowInSearchFilters(true);
|
||||||
|
|
||||||
|
boolean isValid =
|
||||||
|
HidePropertyValidator.validateSettingsUpserts(
|
||||||
|
TestMCP.ofOneUpsertItem(TEST_PROPERTY_URN, propertySettings, TEST_REGISTRY))
|
||||||
|
.findAny()
|
||||||
|
.isEmpty();
|
||||||
|
Assert.assertFalse(isValid);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,160 @@
|
|||||||
|
package com.linkedin.metadata.structuredproperties.validators;
|
||||||
|
|
||||||
|
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME;
|
||||||
|
|
||||||
|
import com.linkedin.common.urn.Urn;
|
||||||
|
import com.linkedin.common.urn.UrnUtils;
|
||||||
|
import com.linkedin.metadata.aspect.GraphRetriever;
|
||||||
|
import com.linkedin.metadata.aspect.RetrieverContext;
|
||||||
|
import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
|
||||||
|
import com.linkedin.metadata.entity.SearchRetriever;
|
||||||
|
import com.linkedin.metadata.models.registry.EntityRegistry;
|
||||||
|
import com.linkedin.metadata.query.filter.Filter;
|
||||||
|
import com.linkedin.metadata.search.ScrollResult;
|
||||||
|
import com.linkedin.metadata.search.SearchEntity;
|
||||||
|
import com.linkedin.metadata.search.SearchEntityArray;
|
||||||
|
import com.linkedin.metadata.structuredproperties.validation.ShowPropertyAsBadgeValidator;
|
||||||
|
import com.linkedin.structured.StructuredPropertySettings;
|
||||||
|
import com.linkedin.test.metadata.aspect.MockAspectRetriever;
|
||||||
|
import com.linkedin.test.metadata.aspect.TestEntityRegistry;
|
||||||
|
import com.linkedin.test.metadata.aspect.batch.TestMCP;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.stream.Stream;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
|
||||||
|
import org.testng.Assert;
|
||||||
|
import org.testng.annotations.BeforeMethod;
|
||||||
|
import org.testng.annotations.Test;
|
||||||
|
|
||||||
|
public class ShowPropertyAsBadgeValidatorTest {
|
||||||
|
|
||||||
|
private static final EntityRegistry TEST_REGISTRY = new TestEntityRegistry();
|
||||||
|
private static final Urn TEST_PROPERTY_URN =
|
||||||
|
UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.privacy.retentionTime");
|
||||||
|
private static final Urn EXISTING_BADGE_URN =
|
||||||
|
UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.privacy.existingBadge");
|
||||||
|
|
||||||
|
private SearchRetriever mockSearchRetriever;
|
||||||
|
private MockAspectRetriever mockAspectRetriever;
|
||||||
|
private GraphRetriever mockGraphRetriever;
|
||||||
|
private RetrieverContext retrieverContext;
|
||||||
|
|
||||||
|
@BeforeMethod
|
||||||
|
public void setup() {
|
||||||
|
mockSearchRetriever = Mockito.mock(SearchRetriever.class);
|
||||||
|
|
||||||
|
StructuredPropertySettings propertySettings =
|
||||||
|
new StructuredPropertySettings()
|
||||||
|
.setShowAsAssetBadge(true)
|
||||||
|
.setShowInAssetSummary(true)
|
||||||
|
.setShowInSearchFilters(true);
|
||||||
|
mockAspectRetriever =
|
||||||
|
new MockAspectRetriever(
|
||||||
|
ImmutableMap.of(
|
||||||
|
TEST_PROPERTY_URN,
|
||||||
|
Collections.singletonList(propertySettings),
|
||||||
|
EXISTING_BADGE_URN,
|
||||||
|
Collections.singletonList(propertySettings)));
|
||||||
|
mockGraphRetriever = Mockito.mock(GraphRetriever.class);
|
||||||
|
retrieverContext =
|
||||||
|
io.datahubproject.metadata.context.RetrieverContext.builder()
|
||||||
|
.aspectRetriever(mockAspectRetriever)
|
||||||
|
.searchRetriever(mockSearchRetriever)
|
||||||
|
.graphRetriever(mockGraphRetriever)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testValidUpsert() {
|
||||||
|
|
||||||
|
// Create settings with showAsAssetBadge = true
|
||||||
|
StructuredPropertySettings propertySettings =
|
||||||
|
new StructuredPropertySettings()
|
||||||
|
.setShowAsAssetBadge(true)
|
||||||
|
.setShowInAssetSummary(true)
|
||||||
|
.setShowInSearchFilters(true);
|
||||||
|
|
||||||
|
Mockito.when(
|
||||||
|
mockSearchRetriever.scroll(
|
||||||
|
Mockito.eq(Collections.singletonList(STRUCTURED_PROPERTY_ENTITY_NAME)),
|
||||||
|
Mockito.any(Filter.class),
|
||||||
|
Mockito.eq(null),
|
||||||
|
Mockito.eq(10)))
|
||||||
|
.thenReturn(new ScrollResult().setEntities(new SearchEntityArray()));
|
||||||
|
|
||||||
|
// Test validation
|
||||||
|
Stream<AspectValidationException> validationResult =
|
||||||
|
ShowPropertyAsBadgeValidator.validateSettingsUpserts(
|
||||||
|
TestMCP.ofOneUpsertItem(TEST_PROPERTY_URN, propertySettings, TEST_REGISTRY),
|
||||||
|
retrieverContext);
|
||||||
|
|
||||||
|
// Assert no validation exceptions
|
||||||
|
Assert.assertTrue(validationResult.findAny().isEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testInvalidUpsertWithExistingBadge() {
|
||||||
|
|
||||||
|
// Create settings with showAsAssetBadge = true
|
||||||
|
StructuredPropertySettings propertySettings =
|
||||||
|
new StructuredPropertySettings()
|
||||||
|
.setShowAsAssetBadge(true)
|
||||||
|
.setShowInAssetSummary(true)
|
||||||
|
.setShowInSearchFilters(true);
|
||||||
|
|
||||||
|
// Mock search results with an existing badge
|
||||||
|
SearchEntity existingBadge = new SearchEntity();
|
||||||
|
existingBadge.setEntity(EXISTING_BADGE_URN);
|
||||||
|
ScrollResult mockResult = new ScrollResult();
|
||||||
|
mockResult.setEntities(new SearchEntityArray(Collections.singletonList(existingBadge)));
|
||||||
|
Mockito.when(
|
||||||
|
mockSearchRetriever.scroll(
|
||||||
|
Mockito.eq(Collections.singletonList(STRUCTURED_PROPERTY_ENTITY_NAME)),
|
||||||
|
Mockito.any(Filter.class),
|
||||||
|
Mockito.eq(null),
|
||||||
|
Mockito.eq(10)))
|
||||||
|
.thenReturn(mockResult);
|
||||||
|
|
||||||
|
// Test validation
|
||||||
|
Stream<AspectValidationException> validationResult =
|
||||||
|
ShowPropertyAsBadgeValidator.validateSettingsUpserts(
|
||||||
|
TestMCP.ofOneUpsertItem(TEST_PROPERTY_URN, propertySettings, TEST_REGISTRY),
|
||||||
|
retrieverContext);
|
||||||
|
|
||||||
|
// Assert validation exception exists
|
||||||
|
Assert.assertFalse(validationResult.findAny().isEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testValidUpsertWithShowAsAssetBadgeFalse() {
|
||||||
|
|
||||||
|
// Create settings with showAsAssetBadge = false
|
||||||
|
StructuredPropertySettings propertySettings =
|
||||||
|
new StructuredPropertySettings()
|
||||||
|
.setShowAsAssetBadge(false)
|
||||||
|
.setShowInAssetSummary(true)
|
||||||
|
.setShowInSearchFilters(true);
|
||||||
|
|
||||||
|
// Mock search results with an existing badge (shouldn't matter since we're setting false)
|
||||||
|
SearchEntity existingBadge = new SearchEntity();
|
||||||
|
existingBadge.setEntity(EXISTING_BADGE_URN);
|
||||||
|
ScrollResult mockResult = new ScrollResult();
|
||||||
|
mockResult.setEntities(new SearchEntityArray(Collections.singletonList(existingBadge)));
|
||||||
|
Mockito.when(
|
||||||
|
mockSearchRetriever.scroll(
|
||||||
|
Mockito.eq(Collections.singletonList(STRUCTURED_PROPERTY_ENTITY_NAME)),
|
||||||
|
Mockito.any(Filter.class),
|
||||||
|
Mockito.eq(null),
|
||||||
|
Mockito.eq(10)))
|
||||||
|
.thenReturn(mockResult);
|
||||||
|
|
||||||
|
// Test validation
|
||||||
|
Stream<AspectValidationException> validationResult =
|
||||||
|
ShowPropertyAsBadgeValidator.validateSettingsUpserts(
|
||||||
|
TestMCP.ofOneUpsertItem(TEST_PROPERTY_URN, propertySettings, TEST_REGISTRY),
|
||||||
|
retrieverContext);
|
||||||
|
|
||||||
|
// Assert no validation exceptions
|
||||||
|
Assert.assertTrue(validationResult.findAny().isEmpty());
|
||||||
|
}
|
||||||
|
}
|
@ -4,6 +4,8 @@ import static com.linkedin.metadata.Constants.EDITABLE_SCHEMA_METADATA_ASPECT_NA
|
|||||||
import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_ENTITY_NAME;
|
import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_ENTITY_NAME;
|
||||||
import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME;
|
import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME;
|
||||||
import static com.linkedin.metadata.Constants.SCHEMA_METADATA_ASPECT_NAME;
|
import static com.linkedin.metadata.Constants.SCHEMA_METADATA_ASPECT_NAME;
|
||||||
|
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME;
|
||||||
|
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME;
|
||||||
|
|
||||||
import com.linkedin.metadata.Constants;
|
import com.linkedin.metadata.Constants;
|
||||||
import com.linkedin.metadata.aspect.hooks.IgnoreUnknownMutator;
|
import com.linkedin.metadata.aspect.hooks.IgnoreUnknownMutator;
|
||||||
@ -15,6 +17,8 @@ import com.linkedin.metadata.aspect.validation.ExecutionRequestResultValidator;
|
|||||||
import com.linkedin.metadata.aspect.validation.FieldPathValidator;
|
import com.linkedin.metadata.aspect.validation.FieldPathValidator;
|
||||||
import com.linkedin.metadata.dataproducts.sideeffects.DataProductUnsetSideEffect;
|
import com.linkedin.metadata.dataproducts.sideeffects.DataProductUnsetSideEffect;
|
||||||
import com.linkedin.metadata.schemafields.sideeffects.SchemaFieldSideEffect;
|
import com.linkedin.metadata.schemafields.sideeffects.SchemaFieldSideEffect;
|
||||||
|
import com.linkedin.metadata.structuredproperties.validation.HidePropertyValidator;
|
||||||
|
import com.linkedin.metadata.structuredproperties.validation.ShowPropertyAsBadgeValidator;
|
||||||
import com.linkedin.metadata.timeline.eventgenerator.EntityChangeEventGeneratorRegistry;
|
import com.linkedin.metadata.timeline.eventgenerator.EntityChangeEventGeneratorRegistry;
|
||||||
import com.linkedin.metadata.timeline.eventgenerator.SchemaMetadataChangeEventGenerator;
|
import com.linkedin.metadata.timeline.eventgenerator.SchemaMetadataChangeEventGenerator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@ -149,4 +153,40 @@ public class SpringStandardPluginConfiguration {
|
|||||||
.build()))
|
.build()))
|
||||||
.build());
|
.build());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public AspectPayloadValidator hidePropertyValidator() {
|
||||||
|
return new HidePropertyValidator()
|
||||||
|
.setConfig(
|
||||||
|
AspectPluginConfig.builder()
|
||||||
|
.className(HidePropertyValidator.class.getName())
|
||||||
|
.enabled(true)
|
||||||
|
.supportedOperations(
|
||||||
|
List.of("UPSERT", "UPDATE", "CREATE", "CREATE_ENTITY", "RESTATE", "PATCH"))
|
||||||
|
.supportedEntityAspectNames(
|
||||||
|
List.of(
|
||||||
|
AspectPluginConfig.EntityAspectName.builder()
|
||||||
|
.entityName(STRUCTURED_PROPERTY_ENTITY_NAME)
|
||||||
|
.aspectName(STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME)
|
||||||
|
.build()))
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public AspectPayloadValidator showPropertyAsAssetBadgeValidator() {
|
||||||
|
return new ShowPropertyAsBadgeValidator()
|
||||||
|
.setConfig(
|
||||||
|
AspectPluginConfig.builder()
|
||||||
|
.className(ShowPropertyAsBadgeValidator.class.getName())
|
||||||
|
.enabled(true)
|
||||||
|
.supportedOperations(
|
||||||
|
List.of("UPSERT", "UPDATE", "CREATE", "CREATE_ENTITY", "RESTATE", "PATCH"))
|
||||||
|
.supportedEntityAspectNames(
|
||||||
|
List.of(
|
||||||
|
AspectPluginConfig.EntityAspectName.builder()
|
||||||
|
.entityName(STRUCTURED_PROPERTY_ENTITY_NAME)
|
||||||
|
.aspectName(STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME)
|
||||||
|
.build()))
|
||||||
|
.build());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user