yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-06-27 05:03:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(validation): enable validation trim options (#12712)

Browse Source
This commit is contained in:
david-leifker 2025-02-24 06:30:45 -06:00 committed by GitHub
parent d1494c2252
commit 16ef1ac174
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 268 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java
View File

@ -231,7 +231,7 @@ public class ChangeItemImpl implements ChangeMCP {
aspect =
GenericRecordUtils.deserializeAspect(
mcp.getAspect().getValue(), mcp.getAspect().getContentType(), aspectSpec);
ValidationApiUtils.validateOrThrow(aspect);
ValidationApiUtils.validateTrimOrThrow(aspect);
} catch (ModelConversionException e) {
throw new RuntimeException(
String.format(

4
metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLItemImpl.java
View File

@ -133,7 +133,7 @@ public class MCLItemImpl implements MCLItem {
aspect =
GenericRecordUtils.deserializeAspect(
mcl.getAspect().getValue(), mcl.getAspect().getContentType(), aspectSpec);
ValidationApiUtils.validateOrThrow(aspect);
ValidationApiUtils.validateTrimOrThrow(aspect);
} else {
aspect = null;
}
@ -144,7 +144,7 @@ public class MCLItemImpl implements MCLItem {
mcl.getPreviousAspectValue().getValue(),
mcl.getPreviousAspectValue().getContentType(),
aspectSpec);
ValidationApiUtils.validateOrThrow(prevAspect);
ValidationApiUtils.validateTrimOrThrow(prevAspect);
} else {
prevAspect = null;
}

26
metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java
View File

@ -36,6 +36,17 @@ public class ValidationApiUtils {
});
}
public static void validateTrimOrThrow(RecordTemplate record) {
RecordTemplateValidator.validateTrim(
record,
validationResult -> {
throw new ValidationException(
String.format(
"Failed to validate record with class %s: %s",
record.getClass().getName(), validationResult.getMessages().toString()));
});
}
public static void validateUrn(@Nonnull EntityRegistry entityRegistry, @Nonnull final Urn urn) {
UrnValidationUtil.validateUrn(
entityRegistry,
@ -45,19 +56,6 @@ public class ValidationApiUtils {
System.getenv().getOrDefault(STRICT_URN_VALIDATION_ENABLED, "false"))));
}
/**
* Validates a {@link RecordTemplate} and logs a warning if validation fails.
*
* @param record record to be validated.ailure.
*/
public static void validateOrWarn(RecordTemplate record) {
RecordTemplateValidator.validate(
record,
validationResult -> {
log.warn(String.format("Failed to validate record %s against its schema.", record));
});
}
public static AspectSpec validate(EntitySpec entitySpec, String aspectName) {
if (aspectName == null || aspectName.isEmpty()) {
throw new UnsupportedOperationException(
@ -95,7 +93,7 @@ public class ValidationApiUtils {
EntityApiUtils.buildKeyAspect(entityRegistry, urn), resultFunction, validator);
if (aspect != null) {
RecordTemplateValidator.validate(aspect, resultFunction, validator);
RecordTemplateValidator.validateTrim(aspect, resultFunction, validator);
}
}
}

2
metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java
View File

@ -278,7 +278,7 @@ public class EntityUtils {
// Read Validate
systemAspects.forEach(
systemAspect ->
RecordTemplateValidator.validate(
RecordTemplateValidator.validateTrim(
systemAspect.getRecordTemplate(),
validationFailure ->
log.warn(

2
metadata-io/src/test/java/com/linkedin/metadata/entity/ValidationUtilsTest.java
View File

@ -44,6 +44,8 @@ public class ValidationUtilsTest {
rawMap.put("extraField", 1);
Status status = new Status(rawMap);
assertThrows(ValidationException.class, () -> ValidationApiUtils.validateOrThrow(status));
// this one should work
ValidationApiUtils.validateTrimOrThrow(status);
}
@Test

6
metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java
View File

@ -8,7 +8,7 @@ import static com.linkedin.metadata.authorization.ApiOperation.CREATE;
import static com.linkedin.metadata.authorization.ApiOperation.DELETE;
import static com.linkedin.metadata.authorization.ApiOperation.EXISTS;
import static com.linkedin.metadata.authorization.ApiOperation.READ;
import static com.linkedin.metadata.entity.validation.ValidationApiUtils.validateOrThrow;
import static com.linkedin.metadata.entity.validation.ValidationApiUtils.validateTrimOrThrow;
import static com.linkedin.metadata.entity.validation.ValidationUtils.*;
import static com.linkedin.metadata.resources.restli.RestliConstants.*;
import static com.linkedin.metadata.search.utils.SearchUtils.*;
@ -286,7 +286,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
}
try {
validateOrThrow(entity);
validateTrimOrThrow(entity);
} catch (ValidationException e) {
throw new RestLiServiceException(HttpStatus.S_422_UNPROCESSABLE_ENTITY, e);
}
@ -333,7 +333,7 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
for (Entity entity : entities) {
try {
validateOrThrow(entity);
validateTrimOrThrow(entity);
} catch (ValidationException e) {
throw new RestLiServiceException(HttpStatus.S_422_UNPROCESSABLE_ENTITY, e);
}

2
metadata-utils/build.gradle
View File

@ -34,6 +34,8 @@ dependencies {
testImplementation project(':test-models')
testImplementation project(path: ':test-models', configuration: 'testDataTemplate')
testImplementation externalDependency.testng
testImplementation externalDependency.mockito
testImplementation externalDependency.mockitoInline
testImplementation project(':metadata-operation-context')
constraints {

38
metadata-utils/src/main/java/com/linkedin/metadata/utils/RecordTemplateValidator.java
View File

@ -21,6 +21,12 @@ public class RecordTemplateValidator {
CoercionMode.NORMAL,
UnrecognizedFieldMode.DISALLOW);
private static final ValidationOptions TRIM_VALIDATION_OPTIONS =
new ValidationOptions(
RequiredMode.CAN_BE_ABSENT_IF_HAS_DEFAULT,
CoercionMode.NORMAL,
UnrecognizedFieldMode.TRIM);
private static final UrnValidator URN_VALIDATOR = new UrnValidator();
/**
@ -37,10 +43,25 @@ public class RecordTemplateValidator {
}
}
/**
* Validates a {@link RecordTemplate} and applies a function if validation fails. Extra fields are
* trimmed.
*
* @param record record to be validated.failure.
*/
public static void validateTrim(
RecordTemplate record, Consumer<ValidationResult> onValidationFailure) {
final ValidationResult result =
ValidateDataAgainstSchema.validate(record, TRIM_VALIDATION_OPTIONS, URN_VALIDATOR);
if (!result.isValid()) {
onValidationFailure.accept(result);
}
}
/**
* Validates a {@link RecordTemplate} and applies a function if validation fails
*
* @param record record to be validated.ailure.
* @param record record to be validated.failure.
*/
public static void validate(
RecordTemplate record, Consumer<ValidationResult> onValidationFailure, Validator validator) {
@ -51,5 +72,20 @@ public class RecordTemplateValidator {
}
}
/**
* Validates a {@link RecordTemplate} and applies a function if validation fails Extra fields are
* trimmed.
*
* @param record record to be validated.failure.
*/
public static void validateTrim(
RecordTemplate record, Consumer<ValidationResult> onValidationFailure, Validator validator) {
final ValidationResult result =
ValidateDataAgainstSchema.validate(record, TRIM_VALIDATION_OPTIONS, validator);
if (!result.isValid()) {
onValidationFailure.accept(result);
}
}
private RecordTemplateValidator() {}
}

208
metadata-utils/src/test/java/com/linkedin/metadata/utils/RecordTemplateValidatorTest.java Normal file
View File

@ -0,0 +1,208 @@
package com.linkedin.metadata.utils;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;
import com.linkedin.data.schema.validation.ValidateDataAgainstSchema;
import com.linkedin.data.schema.validation.ValidationOptions;
import com.linkedin.data.schema.validation.ValidationResult;
import com.linkedin.data.schema.validator.Validator;
import com.linkedin.data.template.RecordTemplate;
import java.util.function.Consumer;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
public class RecordTemplateValidatorTest {
@Mock private RecordTemplate mockRecord;
@Mock private ValidationResult mockValidationResult;
@Mock private Consumer<ValidationResult> mockValidationFailureHandler;
@Mock private Validator mockValidator;
@BeforeMethod
public void setup() {
MockitoAnnotations.openMocks(this);
}
@Test
public void testValidate_WhenValidationSucceeds_DoesNotCallFailureHandler() {
// Arrange
try (var mockedStatic = mockStatic(ValidateDataAgainstSchema.class)) {
when(mockValidationResult.isValid()).thenReturn(true);
mockedStatic
.when(
() ->
ValidateDataAgainstSchema.validate(
any(RecordTemplate.class),
any(ValidationOptions.class),
any(Validator.class)))
.thenReturn(mockValidationResult);
// Act
RecordTemplateValidator.validate(mockRecord, mockValidationFailureHandler);
// Assert
verify(mockValidationFailureHandler, never()).accept(any(ValidationResult.class));
}
}
@Test
public void testValidate_WhenValidationFails_CallsFailureHandler() {
// Arrange
try (var mockedStatic = mockStatic(ValidateDataAgainstSchema.class)) {
when(mockValidationResult.isValid()).thenReturn(false);
mockedStatic
.when(
() ->
ValidateDataAgainstSchema.validate(
any(RecordTemplate.class),
any(ValidationOptions.class),
any(Validator.class)))
.thenReturn(mockValidationResult);
// Act
RecordTemplateValidator.validate(mockRecord, mockValidationFailureHandler);
// Assert
verify(mockValidationFailureHandler).accept(mockValidationResult);
}
}
@Test
public void testValidateTrim_WhenValidationSucceeds_DoesNotCallFailureHandler() {
// Arrange
try (var mockedStatic = mockStatic(ValidateDataAgainstSchema.class)) {
when(mockValidationResult.isValid()).thenReturn(true);
mockedStatic
.when(
() ->
ValidateDataAgainstSchema.validate(
any(RecordTemplate.class),
any(ValidationOptions.class),
any(Validator.class)))
.thenReturn(mockValidationResult);
// Act
RecordTemplateValidator.validateTrim(mockRecord, mockValidationFailureHandler);
// Assert
verify(mockValidationFailureHandler, never()).accept(any(ValidationResult.class));
}
}
@Test
public void testValidateTrim_WhenValidationFails_CallsFailureHandler() {
// Arrange
try (var mockedStatic = mockStatic(ValidateDataAgainstSchema.class)) {
when(mockValidationResult.isValid()).thenReturn(false);
mockedStatic
.when(
() ->
ValidateDataAgainstSchema.validate(
any(RecordTemplate.class),
any(ValidationOptions.class),
any(Validator.class)))
.thenReturn(mockValidationResult);
// Act
RecordTemplateValidator.validateTrim(mockRecord, mockValidationFailureHandler);
// Assert
verify(mockValidationFailureHandler).accept(mockValidationResult);
}
}
@Test
public void testValidateWithCustomValidator_WhenValidationSucceeds_DoesNotCallFailureHandler() {
// Arrange
try (var mockedStatic = mockStatic(ValidateDataAgainstSchema.class)) {
when(mockValidationResult.isValid()).thenReturn(true);
mockedStatic
.when(
() ->
ValidateDataAgainstSchema.validate(
any(RecordTemplate.class),
any(ValidationOptions.class),
any(Validator.class)))
.thenReturn(mockValidationResult);
// Act
RecordTemplateValidator.validate(mockRecord, mockValidationFailureHandler, mockValidator);
// Assert
verify(mockValidationFailureHandler, never()).accept(any(ValidationResult.class));
}
}
@Test
public void testValidateWithCustomValidator_WhenValidationFails_CallsFailureHandler() {
// Arrange
try (var mockedStatic = mockStatic(ValidateDataAgainstSchema.class)) {
when(mockValidationResult.isValid()).thenReturn(false);
mockedStatic
.when(
() ->
ValidateDataAgainstSchema.validate(
any(RecordTemplate.class),
any(ValidationOptions.class),
any(Validator.class)))
.thenReturn(mockValidationResult);
// Act
RecordTemplateValidator.validate(mockRecord, mockValidationFailureHandler, mockValidator);
// Assert
verify(mockValidationFailureHandler).accept(mockValidationResult);
}
}
@Test
public void
testValidateTrimWithCustomValidator_WhenValidationSucceeds_DoesNotCallFailureHandler() {
// Arrange
try (var mockedStatic = mockStatic(ValidateDataAgainstSchema.class)) {
when(mockValidationResult.isValid()).thenReturn(true);
mockedStatic
.when(
() ->
ValidateDataAgainstSchema.validate(
any(RecordTemplate.class),
any(ValidationOptions.class),
any(Validator.class)))
.thenReturn(mockValidationResult);
// Act
RecordTemplateValidator.validateTrim(mockRecord, mockValidationFailureHandler, mockValidator);
// Assert
verify(mockValidationFailureHandler, never()).accept(any(ValidationResult.class));
}
}
@Test
public void testValidateTrimWithCustomValidator_WhenValidationFails_CallsFailureHandler() {
// Arrange
try (var mockedStatic = mockStatic(ValidateDataAgainstSchema.class)) {
when(mockValidationResult.isValid()).thenReturn(false);
mockedStatic
.when(
() ->
ValidateDataAgainstSchema.validate(
any(RecordTemplate.class),
any(ValidationOptions.class),
any(Validator.class)))
.thenReturn(mockValidationResult);
// Act
RecordTemplateValidator.validateTrim(mockRecord, mockValidationFailureHandler, mockValidator);
// Assert
verify(mockValidationFailureHandler).accept(mockValidationResult);
}
}
}