feat(structured-properties): immutable flag (#10461)

Co-authored-by: Chris Collins <chriscollins3456@gmail.com>
This commit is contained in:
david-leifker 2024-05-09 13:55:25 -05:00 committed by GitHub
parent c8bb7dd34a
commit e7c7015981
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 377 additions and 20 deletions

View File

@ -60,6 +60,7 @@ public class StructuredPropertyMapper
definition.setQualifiedName(gmsDefinition.getQualifiedName());
definition.setCardinality(
PropertyCardinality.valueOf(gmsDefinition.getCardinality().toString()));
definition.setImmutable(gmsDefinition.isImmutable());
definition.setValueType(createDataTypeEntity(gmsDefinition.getValueType()));
if (gmsDefinition.hasDisplayName()) {
definition.setDisplayName(gmsDefinition.getDisplayName());

View File

@ -75,6 +75,11 @@ type StructuredPropertyDefinition {
Entity types that this structured property can be applied to
"""
entityTypes: [EntityTypeEntity!]!
"""
Whether or not this structured property is immutable
"""
immutable: Boolean!
}
"""

View File

@ -10,7 +10,7 @@ interface Props {
export function EditColumn({ propertyRow }: Props) {
const [isEditModalVisible, setIsEditModalVisible] = useState(false);
if (!propertyRow.structuredProperty) {
if (!propertyRow.structuredProperty || propertyRow.structuredProperty?.definition.immutable) {
return null;
}

View File

@ -1245,6 +1245,7 @@ fragment structuredPropertyFields on StructuredPropertyEntity {
qualifiedName
description
cardinality
immutable
valueType {
info {
type

View File

@ -27,6 +27,7 @@ import com.linkedin.structured.PropertyValue;
import com.linkedin.structured.StructuredProperties;
import com.linkedin.structured.StructuredPropertyDefinition;
import com.linkedin.structured.StructuredPropertyValueAssignment;
import com.linkedin.util.Pair;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
@ -38,9 +39,11 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import lombok.extern.slf4j.Slf4j;
/** A Validator for StructuredProperties Aspect that is attached to entities like Datasets, etc. */
@ -92,20 +95,22 @@ public class StructuredPropertiesValidator extends AspectPayloadValidator {
@Override
protected Stream<AspectValidationException> validatePreCommitAspects(
@Nonnull Collection<ChangeMCP> changeMCPs, @Nonnull RetrieverContext retrieverContext) {
return Stream.empty();
return validateImmutable(
changeMCPs.stream()
.filter(
i ->
ChangeType.DELETE.equals(i.getChangeType())
|| CHANGE_TYPES.contains(i.getChangeType()))
.collect(Collectors.toList()),
retrieverContext.getAspectRetriever());
}
public static Stream<AspectValidationException> validateProposedUpserts(
@Nonnull Collection<BatchItem> mcpItems, @Nonnull AspectRetriever aspectRetriever) {
ValidationExceptionCollection exceptions = ValidationExceptionCollection.newCollection();
// Validate propertyUrns
Set<Urn> validPropertyUrns = validateStructuredPropertyUrns(mcpItems, exceptions);
// Fetch property aspects for further validation
Map<Urn, Map<String, Aspect>> allStructuredPropertiesAspects =
fetchPropertyAspects(validPropertyUrns, aspectRetriever);
fetchPropertyAspects(mcpItems, aspectRetriever, exceptions, false);
// Validate assignments
for (BatchItem i : exceptions.successful(mcpItems)) {
@ -120,15 +125,13 @@ public class StructuredPropertiesValidator extends AspectPayloadValidator {
softDeleteCheck(i, propertyAspects, "Cannot apply a soft deleted Structured Property value")
.ifPresent(exceptions::addException);
Aspect structuredPropertyDefinitionAspect =
propertyAspects.get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME);
if (structuredPropertyDefinitionAspect == null) {
StructuredPropertyDefinition structuredPropertyDefinition =
lookupPropertyDefinition(propertyUrn, allStructuredPropertiesAspects);
if (structuredPropertyDefinition == null) {
exceptions.addException(i, "Unexpected null value found.");
}
StructuredPropertyDefinition structuredPropertyDefinition =
new StructuredPropertyDefinition(structuredPropertyDefinitionAspect.data());
log.warn(
log.debug(
"Retrieved property definition for {}. {}", propertyUrn, structuredPropertyDefinition);
if (structuredPropertyDefinition != null) {
PrimitivePropertyValueArray values = structuredPropertyValueAssignment.getValues();
@ -158,8 +161,73 @@ public class StructuredPropertiesValidator extends AspectPayloadValidator {
return exceptions.streamAllExceptions();
}
public static Stream<AspectValidationException> validateImmutable(
@Nonnull Collection<ChangeMCP> changeMCPs, @Nonnull AspectRetriever aspectRetriever) {
ValidationExceptionCollection exceptions = ValidationExceptionCollection.newCollection();
final Map<Urn, Map<String, Aspect>> allStructuredPropertiesAspects =
fetchPropertyAspects(changeMCPs, aspectRetriever, exceptions, true);
Set<Urn> immutablePropertyUrns =
allStructuredPropertiesAspects.keySet().stream()
.map(
stringAspectMap ->
Pair.of(
stringAspectMap,
lookupPropertyDefinition(stringAspectMap, allStructuredPropertiesAspects)))
.filter(defPair -> defPair.getSecond() != null && defPair.getSecond().isImmutable())
.map(Pair::getFirst)
.collect(Collectors.toSet());
// Validate immutable assignments
for (ChangeMCP i : exceptions.successful(changeMCPs)) {
// only apply immutable validation if previous properties exist
if (i.getPreviousRecordTemplate() != null) {
Map<Urn, StructuredPropertyValueAssignment> newImmutablePropertyMap =
i.getAspect(StructuredProperties.class).getProperties().stream()
.filter(assign -> immutablePropertyUrns.contains(assign.getPropertyUrn()))
.collect(
Collectors.toMap(
StructuredPropertyValueAssignment::getPropertyUrn, Function.identity()));
Map<Urn, StructuredPropertyValueAssignment> oldImmutablePropertyMap =
i.getPreviousAspect(StructuredProperties.class).getProperties().stream()
.filter(assign -> immutablePropertyUrns.contains(assign.getPropertyUrn()))
.collect(
Collectors.toMap(
StructuredPropertyValueAssignment::getPropertyUrn, Function.identity()));
// upsert/mutation path
newImmutablePropertyMap
.entrySet()
.forEach(
entry -> {
Urn propertyUrn = entry.getKey();
StructuredPropertyValueAssignment assignment = entry.getValue();
if (oldImmutablePropertyMap.containsKey(propertyUrn)
&& !oldImmutablePropertyMap.get(propertyUrn).equals(assignment)) {
exceptions.addException(
i, String.format("Cannot mutate an immutable property: %s", propertyUrn));
}
});
// delete path
oldImmutablePropertyMap.entrySet().stream()
.filter(entry -> !newImmutablePropertyMap.containsKey(entry.getKey()))
.forEach(
entry ->
exceptions.addException(
i,
String.format("Cannot delete an immutable property %s", entry.getKey())));
}
}
return exceptions.streamAllExceptions();
}
private static Set<Urn> validateStructuredPropertyUrns(
Collection<BatchItem> mcpItems, ValidationExceptionCollection exceptions) {
Collection<? extends BatchItem> mcpItems, ValidationExceptionCollection exceptions) {
Set<Urn> validPropertyUrns = new HashSet<>();
for (BatchItem i : exceptions.successful(mcpItems)) {
@ -202,6 +270,17 @@ public class StructuredPropertiesValidator extends AspectPayloadValidator {
return validPropertyUrns;
}
private static Set<Urn> previousStructuredPropertyUrns(Collection<? extends BatchItem> mcpItems) {
return mcpItems.stream()
.filter(i -> i instanceof ChangeMCP)
.map(i -> ((ChangeMCP) i))
.filter(i -> i.getPreviousRecordTemplate() != null)
.flatMap(i -> i.getPreviousAspect(StructuredProperties.class).getProperties().stream())
.map(StructuredPropertyValueAssignment::getPropertyUrn)
.filter(propertyUrn -> propertyUrn.getEntityType().equals("structuredProperty"))
.collect(Collectors.toSet());
}
private static Optional<AspectValidationException> validateAllowedValues(
BatchItem item,
Urn propertyUrn,
@ -338,14 +417,40 @@ public class StructuredPropertiesValidator extends AspectPayloadValidator {
}
private static Map<Urn, Map<String, Aspect>> fetchPropertyAspects(
Set<Urn> structuredPropertyUrns, AspectRetriever aspectRetriever) {
if (structuredPropertyUrns.isEmpty()) {
@Nonnull Collection<? extends BatchItem> mcpItems,
AspectRetriever aspectRetriever,
@Nonnull ValidationExceptionCollection exceptions,
boolean includePrevious) {
// Validate propertyUrns
Set<Urn> validPropertyUrns =
Stream.concat(
validateStructuredPropertyUrns(mcpItems, exceptions).stream(),
includePrevious
? previousStructuredPropertyUrns(mcpItems).stream()
: Stream.empty())
.collect(Collectors.toSet());
if (validPropertyUrns.isEmpty()) {
return Collections.emptyMap();
} else {
return aspectRetriever.getLatestAspectObjects(
structuredPropertyUrns,
validPropertyUrns,
ImmutableSet.of(
Constants.STATUS_ASPECT_NAME, STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME));
}
}
@Nullable
private static StructuredPropertyDefinition lookupPropertyDefinition(
@Nonnull Urn propertyUrn,
@Nonnull Map<Urn, Map<String, Aspect>> allStructuredPropertiesAspects) {
Map<String, Aspect> propertyAspects =
allStructuredPropertiesAspects.getOrDefault(propertyUrn, Collections.emptyMap());
Aspect structuredPropertyDefinitionAspect =
propertyAspects.get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME);
return structuredPropertyDefinitionAspect == null
? null
: new StructuredPropertyDefinition(structuredPropertyDefinitionAspect.data());
}
}

View File

@ -4,6 +4,9 @@ import static org.testng.Assert.assertEquals;
import com.linkedin.common.Status;
import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.events.metadata.ChangeType;
import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
import com.linkedin.metadata.aspect.validation.StructuredPropertiesValidator;
import com.linkedin.metadata.models.registry.EntityRegistry;
import com.linkedin.structured.PrimitivePropertyValue;
@ -19,6 +22,9 @@ import com.linkedin.test.metadata.aspect.TestEntityRegistry;
import com.linkedin.test.metadata.aspect.batch.TestMCP;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.testng.Assert;
import org.testng.annotations.Test;
@ -26,6 +32,9 @@ public class StructuredPropertiesValidatorTest {
private static final EntityRegistry TEST_REGISTRY = new TestEntityRegistry();
private static final Urn TEST_DATASET_URN =
UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:datahub,Test,PROD)");
@Test
public void testValidateAspectNumberUpsert() throws URISyntaxException {
Urn propertyUrn =
@ -268,4 +277,215 @@ public class StructuredPropertiesValidatorTest {
1,
"Should have raised exception for soft deleted definition");
}
@Test
public void testValidateImmutableMutation() throws URISyntaxException {
Urn mutablePropertyUrn =
Urn.createFromString("urn:li:structuredProperty:io.acryl.mutableProperty");
StructuredPropertyDefinition mutablePropertyDef =
new StructuredPropertyDefinition()
.setImmutable(false)
.setValueType(Urn.createFromString("urn:li:type:datahub.number"))
.setAllowedValues(
new PropertyValueArray(
List.of(
new PropertyValue().setValue(PrimitivePropertyValue.create(30.0)),
new PropertyValue().setValue(PrimitivePropertyValue.create(60.0)),
new PropertyValue().setValue(PrimitivePropertyValue.create(90.0)))));
StructuredPropertyValueAssignment mutableAssignment =
new StructuredPropertyValueAssignment()
.setPropertyUrn(mutablePropertyUrn)
.setValues(new PrimitivePropertyValueArray(PrimitivePropertyValue.create(30.0)));
StructuredProperties mutablePayload =
new StructuredProperties()
.setProperties(new StructuredPropertyValueAssignmentArray(mutableAssignment));
Urn immutablePropertyUrn =
Urn.createFromString("urn:li:structuredProperty:io.acryl.immutableProperty");
StructuredPropertyDefinition immutablePropertyDef =
new StructuredPropertyDefinition()
.setImmutable(true)
.setValueType(Urn.createFromString("urn:li:type:datahub.number"))
.setAllowedValues(
new PropertyValueArray(
List.of(
new PropertyValue().setValue(PrimitivePropertyValue.create(30.0)),
new PropertyValue().setValue(PrimitivePropertyValue.create(60.0)),
new PropertyValue().setValue(PrimitivePropertyValue.create(90.0)))));
StructuredPropertyValueAssignment immutableAssignment =
new StructuredPropertyValueAssignment()
.setPropertyUrn(immutablePropertyUrn)
.setValues(new PrimitivePropertyValueArray(PrimitivePropertyValue.create(30.0)));
StructuredProperties immutablePayload =
new StructuredProperties()
.setProperties(new StructuredPropertyValueAssignmentArray(immutableAssignment));
// No previous values for either
boolean noPreviousValid =
StructuredPropertiesValidator.validateImmutable(
Stream.concat(
TestMCP.ofOneMCP(TEST_DATASET_URN, null, mutablePayload, TEST_REGISTRY)
.stream(),
TestMCP.ofOneMCP(
TEST_DATASET_URN, null, immutablePayload, TEST_REGISTRY)
.stream())
.collect(Collectors.toSet()),
new MockAspectRetriever(
Map.of(
mutablePropertyUrn,
List.of(mutablePropertyDef),
immutablePropertyUrn,
List.of(immutablePropertyDef))))
.count()
== 0;
Assert.assertTrue(noPreviousValid);
// Unchanged values of previous (no issues with immutability)
boolean noChangeValid =
StructuredPropertiesValidator.validateImmutable(
Stream.concat(
TestMCP.ofOneMCP(
TEST_DATASET_URN, mutablePayload, mutablePayload, TEST_REGISTRY)
.stream(),
TestMCP.ofOneMCP(
TEST_DATASET_URN, immutablePayload, immutablePayload, TEST_REGISTRY)
.stream())
.collect(Collectors.toSet()),
new MockAspectRetriever(
Map.of(
mutablePropertyUrn,
List.of(mutablePropertyDef),
immutablePropertyUrn,
List.of(immutablePropertyDef))))
.count()
== 0;
Assert.assertTrue(noChangeValid);
// invalid
StructuredPropertyValueAssignment immutableAssignment2 =
new StructuredPropertyValueAssignment()
.setPropertyUrn(immutablePropertyUrn)
.setValues(new PrimitivePropertyValueArray(PrimitivePropertyValue.create(60.0)));
StructuredProperties immutablePayload2 =
new StructuredProperties()
.setProperties(new StructuredPropertyValueAssignmentArray(immutableAssignment2));
List<AspectValidationException> exceptions =
StructuredPropertiesValidator.validateImmutable(
Stream.concat(
TestMCP.ofOneMCP(
TEST_DATASET_URN, mutablePayload, mutablePayload, TEST_REGISTRY)
.stream(),
TestMCP.ofOneMCP(
TEST_DATASET_URN, immutablePayload, immutablePayload2, TEST_REGISTRY)
.stream())
.collect(Collectors.toSet()),
new MockAspectRetriever(
Map.of(
mutablePropertyUrn,
List.of(mutablePropertyDef),
immutablePropertyUrn,
List.of(immutablePropertyDef))))
.collect(Collectors.toList());
Assert.assertEquals(exceptions.size(), 1, "Expected rejected mutation of immutable property.");
Assert.assertEquals(exceptions.get(0).getExceptionKey().getKey(), TEST_DATASET_URN);
Assert.assertTrue(
exceptions.get(0).getMessage().contains("Cannot mutate an immutable property"));
}
@Test
public void testValidateImmutableDelete() throws URISyntaxException {
final StructuredProperties emptyProperties =
new StructuredProperties().setProperties(new StructuredPropertyValueAssignmentArray());
Urn mutablePropertyUrn =
Urn.createFromString("urn:li:structuredProperty:io.acryl.mutableProperty");
StructuredPropertyDefinition mutablePropertyDef =
new StructuredPropertyDefinition()
.setImmutable(false)
.setValueType(Urn.createFromString("urn:li:type:datahub.number"))
.setAllowedValues(
new PropertyValueArray(
List.of(
new PropertyValue().setValue(PrimitivePropertyValue.create(30.0)),
new PropertyValue().setValue(PrimitivePropertyValue.create(60.0)),
new PropertyValue().setValue(PrimitivePropertyValue.create(90.0)))));
StructuredPropertyValueAssignment mutableAssignment =
new StructuredPropertyValueAssignment()
.setPropertyUrn(mutablePropertyUrn)
.setValues(new PrimitivePropertyValueArray(PrimitivePropertyValue.create(30.0)));
StructuredProperties mutablePayload =
new StructuredProperties()
.setProperties(new StructuredPropertyValueAssignmentArray(mutableAssignment));
Urn immutablePropertyUrn =
Urn.createFromString("urn:li:structuredProperty:io.acryl.immutableProperty");
StructuredPropertyDefinition immutablePropertyDef =
new StructuredPropertyDefinition()
.setImmutable(true)
.setValueType(Urn.createFromString("urn:li:type:datahub.number"))
.setAllowedValues(
new PropertyValueArray(
List.of(
new PropertyValue().setValue(PrimitivePropertyValue.create(30.0)),
new PropertyValue().setValue(PrimitivePropertyValue.create(60.0)),
new PropertyValue().setValue(PrimitivePropertyValue.create(90.0)))));
StructuredPropertyValueAssignment immutableAssignment =
new StructuredPropertyValueAssignment()
.setPropertyUrn(immutablePropertyUrn)
.setValues(new PrimitivePropertyValueArray(PrimitivePropertyValue.create(30.0)));
StructuredProperties immutablePayload =
new StructuredProperties()
.setProperties(new StructuredPropertyValueAssignmentArray(immutableAssignment));
// Delete mutable, Delete with no-op for immutable allowed
boolean noPreviousValid =
StructuredPropertiesValidator.validateImmutable(
Stream.concat(
TestMCP.ofOneMCP(
TEST_DATASET_URN, mutablePayload, emptyProperties, TEST_REGISTRY)
.stream(),
TestMCP.ofOneMCP(
TEST_DATASET_URN, immutablePayload, immutablePayload, TEST_REGISTRY)
.stream())
// set to DELETE
.map(i -> ((TestMCP) i).toBuilder().changeType(ChangeType.DELETE).build())
.collect(Collectors.toSet()),
new MockAspectRetriever(
Map.of(
mutablePropertyUrn,
List.of(mutablePropertyDef),
immutablePropertyUrn,
List.of(immutablePropertyDef))))
.count()
== 0;
Assert.assertTrue(noPreviousValid);
// invalid (delete of mutable allowed, delete of immutable denied)
List<AspectValidationException> exceptions =
StructuredPropertiesValidator.validateImmutable(
Stream.concat(
TestMCP.ofOneMCP(
TEST_DATASET_URN, mutablePayload, emptyProperties, TEST_REGISTRY)
.stream(),
TestMCP.ofOneMCP(
TEST_DATASET_URN, immutablePayload, emptyProperties, TEST_REGISTRY)
.stream())
// set to DELETE
.map(i -> ((TestMCP) i).toBuilder().changeType(ChangeType.DELETE).build())
.collect(Collectors.toSet()),
new MockAspectRetriever(
Map.of(
mutablePropertyUrn,
List.of(mutablePropertyDef),
immutablePropertyUrn,
List.of(immutablePropertyDef))))
.collect(Collectors.toList());
Assert.assertEquals(exceptions.size(), 1, "Expected rejected delete of immutable property.");
Assert.assertEquals(exceptions.get(0).getExceptionKey().getKey(), TEST_DATASET_URN);
Assert.assertTrue(
exceptions.get(0).getMessage().contains("Cannot delete an immutable property"));
}
}

View File

@ -27,7 +27,7 @@ import lombok.Builder;
import lombok.Getter;
import lombok.Setter;
@Builder
@Builder(toBuilder = true)
@Getter
public class TestMCP implements ChangeMCP {
private static final String TEST_DATASET_URN =

View File

@ -75,6 +75,7 @@ class StructuredProperties(ConfigModel):
cardinality: Optional[str] = None
allowed_values: Optional[List[AllowedValue]] = None
type_qualifier: Optional[TypeQualifierAllowedTypes] = None
immutable: Optional[bool] = False
@property
def fqn(self) -> str:
@ -124,6 +125,7 @@ class StructuredProperties(ConfigModel):
for entity_type in structuredproperty.entity_types or []
],
cardinality=structuredproperty.cardinality,
immutable=structuredproperty.immutable,
allowedValues=[
PropertyValueClass(
value=v.value, description=v.description

View File

@ -70,5 +70,13 @@ record StructuredPropertyDefinition {
* from the logical type.
*/
searchConfiguration: optional DataHubSearchConfig
/**
* Whether the structured property value is immutable once applied to an entity.
*/
@Searchable = {
"fieldType": "BOOLEAN"
}
immutable: boolean = false
}

View File

@ -588,6 +588,7 @@ plugins:
supportedOperations:
- CREATE
- UPSERT
- DELETE
supportedEntityAspectNames:
- entityName: '*'
aspectName: structuredProperties

View File

@ -653,6 +653,19 @@ public class PoliciesConfig {
CREATE_ENTITY_PRIVILEGE,
EXISTS_ENTITY_PRIVILEGE));
// Properties Privileges
public static final ResourcePrivileges STRUCTURED_PROPERTIES_PRIVILEGES =
ResourcePrivileges.of(
"structuredProperty",
"Structured Properties",
"Structured Properties",
ImmutableList.of(
CREATE_ENTITY_PRIVILEGE,
VIEW_ENTITY_PAGE_PRIVILEGE,
EXISTS_ENTITY_PRIVILEGE,
EDIT_ENTITY_PRIVILEGE,
DELETE_ENTITY_PRIVILEGE));
// ERModelRelationship Privileges
public static final ResourcePrivileges ER_MODEL_RELATIONSHIP_PRIVILEGES =
ResourcePrivileges.of(
@ -689,7 +702,8 @@ public class PoliciesConfig {
NOTEBOOK_PRIVILEGES,
DATA_PRODUCT_PRIVILEGES,
ER_MODEL_RELATIONSHIP_PRIVILEGES,
BUSINESS_ATTRIBUTE_PRIVILEGES);
BUSINESS_ATTRIBUTE_PRIVILEGES,
STRUCTURED_PROPERTIES_PRIVILEGES);
// Merge all entity specific resource privileges to create a superset of all resource privileges
public static final ResourcePrivileges ALL_RESOURCE_PRIVILEGES =