mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-12-04 19:44:58 +00:00
MINOR - Handle chart for Data Contract and add schema validations (#23242)
* test for chart * MINOR - Handle chart for Data Contract and add schema validations * MINOR - Handle chart for Data Contract and add schema validations
This commit is contained in:
Pere Miquel Brull
GitHub
parent
49fd36976c
commit
17d5a5d83f
@ -228,6 +228,12 @@ public class DataContractRepository extends EntityRepository<DataContract> {
|
||||
case Entity.TOPIC:
|
||||
failedFields = validateFieldsAgainstTopic(dataContract, entityRef);
|
||||
break;
|
||||
case Entity.API_ENDPOINT:
|
||||
failedFields = validateFieldsAgainstApiEndpoint(dataContract, entityRef);
|
||||
break;
|
||||
case Entity.DASHBOARD_DATA_MODEL:
|
||||
failedFields = validateFieldsAgainstDashboardDataModel(dataContract, entityRef);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
@ -245,52 +251,98 @@ public class DataContractRepository extends EntityRepository<DataContract> {
|
||||
|
||||
private List<String> validateFieldsAgainstTable(
|
||||
DataContract dataContract, EntityReference tableRef) {
|
||||
List<String> failedFields = new ArrayList<>();
|
||||
org.openmetadata.schema.entity.data.Table table =
|
||||
Entity.getEntity(Entity.TABLE, tableRef.getId(), "columns", Include.NON_DELETED);
|
||||
|
||||
if (table.getColumns() == null || table.getColumns().isEmpty()) {
|
||||
// If table has no columns, all contract fields fail validation
|
||||
return dataContract.getSchema().stream()
|
||||
.map(org.openmetadata.schema.type.Column::getName)
|
||||
.collect(Collectors.toList());
|
||||
return getAllContractFieldNames(dataContract);
|
||||
}
|
||||
|
||||
Set<String> tableColumnNames =
|
||||
table.getColumns().stream().map(Column::getName).collect(Collectors.toSet());
|
||||
|
||||
for (org.openmetadata.schema.type.Column column : dataContract.getSchema()) {
|
||||
if (!tableColumnNames.contains(column.getName())) {
|
||||
failedFields.add(column.getName());
|
||||
}
|
||||
}
|
||||
|
||||
return failedFields;
|
||||
return validateContractFieldsAgainstNames(dataContract, tableColumnNames);
|
||||
}
|
||||
|
||||
private List<String> validateFieldsAgainstTopic(
|
||||
DataContract dataContract, EntityReference topicRef) {
|
||||
List<String> failedFields = new ArrayList<>();
|
||||
Topic topic =
|
||||
Entity.getEntity(Entity.TOPIC, topicRef.getId(), "messageSchema", Include.NON_DELETED);
|
||||
|
||||
if (topic.getMessageSchema() == null
|
||||
|| topic.getMessageSchema().getSchemaFields() == null
|
||||
|| topic.getMessageSchema().getSchemaFields().isEmpty()) {
|
||||
// If topic has no schema, all contract fields fail validation
|
||||
return dataContract.getSchema().stream()
|
||||
.map(org.openmetadata.schema.type.Column::getName)
|
||||
.collect(Collectors.toList());
|
||||
return getAllContractFieldNames(dataContract);
|
||||
}
|
||||
|
||||
Set<String> topicFieldNames = extractFieldNames(topic.getMessageSchema().getSchemaFields());
|
||||
|
||||
return validateContractFieldsAgainstNames(dataContract, topicFieldNames);
|
||||
}
|
||||
|
||||
private List<String> validateFieldsAgainstApiEndpoint(
|
||||
DataContract dataContract, EntityReference apiEndpointRef) {
|
||||
org.openmetadata.schema.entity.data.APIEndpoint apiEndpoint =
|
||||
Entity.getEntity(
|
||||
Entity.API_ENDPOINT,
|
||||
apiEndpointRef.getId(),
|
||||
"requestSchema,responseSchema",
|
||||
Include.NON_DELETED);
|
||||
|
||||
Set<String> apiFieldNames = new HashSet<>();
|
||||
|
||||
if (apiEndpoint.getRequestSchema() != null
|
||||
&& apiEndpoint.getRequestSchema().getSchemaFields() != null
|
||||
&& !apiEndpoint.getRequestSchema().getSchemaFields().isEmpty()) {
|
||||
apiFieldNames.addAll(extractFieldNames(apiEndpoint.getRequestSchema().getSchemaFields()));
|
||||
}
|
||||
|
||||
if (apiEndpoint.getResponseSchema() != null
|
||||
&& apiEndpoint.getResponseSchema().getSchemaFields() != null
|
||||
&& !apiEndpoint.getResponseSchema().getSchemaFields().isEmpty()) {
|
||||
apiFieldNames.addAll(extractFieldNames(apiEndpoint.getResponseSchema().getSchemaFields()));
|
||||
}
|
||||
|
||||
if (apiFieldNames.isEmpty()) {
|
||||
return getAllContractFieldNames(dataContract);
|
||||
}
|
||||
|
||||
return validateContractFieldsAgainstNames(dataContract, apiFieldNames);
|
||||
}
|
||||
|
||||
private List<String> validateFieldsAgainstDashboardDataModel(
|
||||
DataContract dataContract, EntityReference dashboardDataModelRef) {
|
||||
org.openmetadata.schema.entity.data.DashboardDataModel dashboardDataModel =
|
||||
Entity.getEntity(
|
||||
Entity.DASHBOARD_DATA_MODEL,
|
||||
dashboardDataModelRef.getId(),
|
||||
"columns",
|
||||
Include.NON_DELETED);
|
||||
|
||||
if (dashboardDataModel.getColumns() == null || dashboardDataModel.getColumns().isEmpty()) {
|
||||
return getAllContractFieldNames(dataContract);
|
||||
}
|
||||
|
||||
Set<String> dataModelColumnNames =
|
||||
dashboardDataModel.getColumns().stream().map(Column::getName).collect(Collectors.toSet());
|
||||
|
||||
return validateContractFieldsAgainstNames(dataContract, dataModelColumnNames);
|
||||
}
|
||||
|
||||
private List<String> getAllContractFieldNames(DataContract dataContract) {
|
||||
return dataContract.getSchema().stream()
|
||||
.map(org.openmetadata.schema.type.Column::getName)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private List<String> validateContractFieldsAgainstNames(
|
||||
DataContract dataContract, Set<String> entityFieldNames) {
|
||||
List<String> failedFields = new ArrayList<>();
|
||||
for (org.openmetadata.schema.type.Column column : dataContract.getSchema()) {
|
||||
if (!topicFieldNames.contains(column.getName())) {
|
||||
if (!entityFieldNames.contains(column.getName())) {
|
||||
failedFields.add(column.getName());
|
||||
}
|
||||
}
|
||||
|
||||
return failedFields;
|
||||
}
|
||||
|
||||
@ -369,6 +421,7 @@ public class DataContractRepository extends EntityRepository<DataContract> {
|
||||
Entity.DATABASE,
|
||||
Entity.DATABASE_SCHEMA,
|
||||
Entity.DASHBOARD,
|
||||
Entity.CHART,
|
||||
Entity.DASHBOARD_DATA_MODEL,
|
||||
Entity.PIPELINE,
|
||||
Entity.TOPIC,
|
||||
|
||||
@ -71,6 +71,7 @@ import org.openmetadata.schema.api.services.CreateMessagingService;
|
||||
import org.openmetadata.schema.api.services.DatabaseConnection;
|
||||
import org.openmetadata.schema.api.tests.CreateTestCase;
|
||||
import org.openmetadata.schema.entity.data.APIEndpoint;
|
||||
import org.openmetadata.schema.entity.data.Chart;
|
||||
import org.openmetadata.schema.entity.data.Dashboard;
|
||||
import org.openmetadata.schema.entity.data.DashboardDataModel;
|
||||
import org.openmetadata.schema.entity.data.DataContract;
|
||||
@ -108,12 +109,14 @@ import org.openmetadata.sdk.PipelineServiceClientInterface;
|
||||
import org.openmetadata.service.jdbi3.DataContractRepository;
|
||||
import org.openmetadata.service.resources.EntityResourceTest;
|
||||
import org.openmetadata.service.resources.apis.APIEndpointResourceTest;
|
||||
import org.openmetadata.service.resources.charts.ChartResourceTest;
|
||||
import org.openmetadata.service.resources.dashboards.DashboardResourceTest;
|
||||
import org.openmetadata.service.resources.databases.TableResourceTest;
|
||||
import org.openmetadata.service.resources.datamodels.DashboardDataModelResourceTest;
|
||||
import org.openmetadata.service.resources.dqtests.TestCaseResourceTest;
|
||||
import org.openmetadata.service.resources.dqtests.TestSuiteResourceTest;
|
||||
import org.openmetadata.service.resources.services.ingestionpipelines.IngestionPipelineResourceTest;
|
||||
import org.openmetadata.service.resources.topics.TopicResourceTest;
|
||||
import org.openmetadata.service.security.SecurityUtil;
|
||||
import org.openmetadata.service.util.TestUtils;
|
||||
|
||||
@ -138,6 +141,7 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
private static TestCaseResourceTest testCaseResourceTest;
|
||||
private static IngestionPipelineResourceTest ingestionPipelineResourceTest;
|
||||
private static TestSuiteResourceTest testSuiteResourceTest;
|
||||
private static ChartResourceTest chartResourceTest;
|
||||
private static DashboardResourceTest dashboardResourceTest;
|
||||
private static APIEndpointResourceTest apiEndpointResourceTest;
|
||||
private static DashboardDataModelResourceTest dashboardDataModelResourceTest;
|
||||
@ -160,6 +164,7 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
public void setup(TestInfo test) throws URISyntaxException, IOException {
|
||||
testCaseResourceTest = new TestCaseResourceTest();
|
||||
testSuiteResourceTest = new TestSuiteResourceTest();
|
||||
chartResourceTest = new ChartResourceTest();
|
||||
dashboardResourceTest = new DashboardResourceTest();
|
||||
apiEndpointResourceTest = new APIEndpointResourceTest();
|
||||
dashboardDataModelResourceTest = new DashboardDataModelResourceTest();
|
||||
@ -588,6 +593,13 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
* Creates a unique topic for testing data contracts
|
||||
*/
|
||||
private Topic createUniqueTopic(String testName) throws IOException {
|
||||
return createUniqueTopic(testName, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a unique topic for testing data contracts with specific schema fields
|
||||
*/
|
||||
private Topic createUniqueTopic(String testName, List<Field> customFields) throws IOException {
|
||||
// Ensure we have a messaging service to work with
|
||||
String messagingServiceName = ensureMessagingService();
|
||||
|
||||
@ -610,24 +622,27 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
+ uniqueId.substring(0, 8);
|
||||
|
||||
// Create message schema for the topic
|
||||
List<Field> fields =
|
||||
List.of(
|
||||
new Field()
|
||||
.withName("messageId")
|
||||
.withDisplayName("Message ID")
|
||||
.withDataType(FieldDataType.STRING),
|
||||
new Field()
|
||||
.withName("eventType")
|
||||
.withDisplayName("Event Type")
|
||||
.withDataType(FieldDataType.STRING),
|
||||
new Field()
|
||||
.withName("payload")
|
||||
.withDisplayName("Payload")
|
||||
.withDataType(FieldDataType.STRING),
|
||||
new Field()
|
||||
.withName("timestamp")
|
||||
.withDisplayName("Timestamp")
|
||||
.withDataType(FieldDataType.TIMESTAMP));
|
||||
List<Field> fields = customFields;
|
||||
if (fields == null) {
|
||||
fields =
|
||||
List.of(
|
||||
new Field()
|
||||
.withName("messageId")
|
||||
.withDisplayName("Message ID")
|
||||
.withDataType(FieldDataType.STRING),
|
||||
new Field()
|
||||
.withName("eventType")
|
||||
.withDisplayName("Event Type")
|
||||
.withDataType(FieldDataType.STRING),
|
||||
new Field()
|
||||
.withName("payload")
|
||||
.withDisplayName("Payload")
|
||||
.withDataType(FieldDataType.STRING),
|
||||
new Field()
|
||||
.withName("timestamp")
|
||||
.withDisplayName("Timestamp")
|
||||
.withDataType(FieldDataType.TIMESTAMP));
|
||||
}
|
||||
|
||||
MessageSchema messageSchema =
|
||||
new MessageSchema()
|
||||
@ -657,6 +672,17 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
* Creates a unique API endpoint for testing data contracts
|
||||
*/
|
||||
private APIEndpoint createUniqueApiEndpoint(String testName) throws IOException {
|
||||
return createUniqueApiEndpoint(testName, null, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a unique API endpoint for testing data contracts with specific schema fields
|
||||
*/
|
||||
private APIEndpoint createUniqueApiEndpoint(
|
||||
String testName,
|
||||
List<org.openmetadata.schema.type.Field> requestFields,
|
||||
List<org.openmetadata.schema.type.Field> responseFields)
|
||||
throws IOException {
|
||||
// Use multiple entropy sources for absolute uniqueness
|
||||
long counter = tableCounter.incrementAndGet();
|
||||
long timestamp = System.nanoTime();
|
||||
@ -675,6 +701,19 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
+ uniqueId;
|
||||
|
||||
CreateAPIEndpoint createApiEndpoint = apiEndpointResourceTest.createRequest(apiEndpointName);
|
||||
|
||||
if (requestFields != null && !requestFields.isEmpty()) {
|
||||
org.openmetadata.schema.type.APISchema requestSchema =
|
||||
new org.openmetadata.schema.type.APISchema().withSchemaFields(requestFields);
|
||||
createApiEndpoint.withRequestSchema(requestSchema);
|
||||
}
|
||||
|
||||
if (responseFields != null && !responseFields.isEmpty()) {
|
||||
org.openmetadata.schema.type.APISchema responseSchema =
|
||||
new org.openmetadata.schema.type.APISchema().withSchemaFields(responseFields);
|
||||
createApiEndpoint.withResponseSchema(responseSchema);
|
||||
}
|
||||
|
||||
WebTarget target = APP.client().target(getApiEndpointUri());
|
||||
Response response =
|
||||
SecurityUtil.addHeaders(target, ADMIN_AUTH_HEADERS).post(Entity.json(createApiEndpoint));
|
||||
@ -688,6 +727,14 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
* Creates a unique dashboard data model for testing data contracts
|
||||
*/
|
||||
private DashboardDataModel createUniqueDashboardDataModel(String testName) throws IOException {
|
||||
return createUniqueDashboardDataModel(testName, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a unique dashboard data model for testing data contracts with specific columns
|
||||
*/
|
||||
private DashboardDataModel createUniqueDashboardDataModel(
|
||||
String testName, List<org.openmetadata.schema.type.Column> columns) throws IOException {
|
||||
// Use multiple entropy sources for absolute uniqueness
|
||||
long counter = tableCounter.incrementAndGet();
|
||||
long timestamp = System.nanoTime();
|
||||
@ -707,6 +754,11 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
|
||||
CreateDashboardDataModel createDataModel =
|
||||
dashboardDataModelResourceTest.createRequest(dataModelName);
|
||||
|
||||
if (columns != null && !columns.isEmpty()) {
|
||||
createDataModel.withColumns(columns);
|
||||
}
|
||||
|
||||
WebTarget target = APP.client().target(getDashboardDataModelUri());
|
||||
Response response =
|
||||
SecurityUtil.addHeaders(target, ADMIN_AUTH_HEADERS).post(Entity.json(createDataModel));
|
||||
@ -3646,6 +3698,97 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
assertEquals(validationResult.getId(), retrievedContract.getLatestResult().getResultId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void testDataContractWithChartAndSemanticsOnly(TestInfo test) throws IOException {
|
||||
// Create a chart entity to use for the data contract
|
||||
Chart chart =
|
||||
chartResourceTest.createEntity(
|
||||
chartResourceTest.createRequest(test.getDisplayName()), ADMIN_AUTH_HEADERS);
|
||||
|
||||
// Test 1: Chart with schema validation should fail (charts don't support schema validation)
|
||||
List<Column> columns =
|
||||
List.of(
|
||||
new Column()
|
||||
.withName("testField")
|
||||
.withDescription("Test field")
|
||||
.withDataType(ColumnDataType.STRING));
|
||||
|
||||
CreateDataContract createWithSchema =
|
||||
createDataContractRequestForEntity(test.getDisplayName() + "_schema", chart)
|
||||
.withSchema(columns);
|
||||
|
||||
assertResponseContains(
|
||||
() -> createDataContract(createWithSchema),
|
||||
BAD_REQUEST,
|
||||
"Schema validation is not supported for chart entities. Only table, topic, apiEndpoint, and dashboardDataModel entities support schema validation");
|
||||
|
||||
// Test 2: Chart with semantics validation only should succeed
|
||||
// Create semantics rules only (no quality expectations or schema)
|
||||
List<SemanticsRule> semanticsRules =
|
||||
List.of(
|
||||
new SemanticsRule()
|
||||
.withName("Chart Display Name Check")
|
||||
.withDescription("Ensures chart has a valid display name")
|
||||
.withRule("{ \"!!\": { \"var\": \"displayName\" } }"),
|
||||
new SemanticsRule()
|
||||
.withName("Chart Type Check")
|
||||
.withDescription("Ensures chart has a valid chart type")
|
||||
.withRule("{ \"!!\": { \"var\": \"chartType\" } }"));
|
||||
|
||||
// Create data contract for the chart with semantics rules only
|
||||
CreateDataContract create =
|
||||
createDataContractRequestForEntity(test.getDisplayName(), chart)
|
||||
.withDescription("Data contract for chart with semantics validation")
|
||||
.withSemantics(semanticsRules)
|
||||
.withEntityStatus(EntityStatus.APPROVED);
|
||||
|
||||
DataContract dataContract = createDataContract(create);
|
||||
|
||||
// Verify the data contract was created successfully
|
||||
assertNotNull(dataContract);
|
||||
assertNotNull(dataContract.getId());
|
||||
assertEquals(create.getName(), dataContract.getName());
|
||||
assertEquals(create.getEntityStatus(), dataContract.getEntityStatus());
|
||||
assertEquals(chart.getId(), dataContract.getEntity().getId());
|
||||
assertEquals("chart", dataContract.getEntity().getType());
|
||||
|
||||
// Verify semantics rules are properly set
|
||||
assertNotNull(dataContract.getSemantics());
|
||||
assertEquals(2, dataContract.getSemantics().size());
|
||||
assertSemantics(create.getSemantics(), dataContract.getSemantics());
|
||||
|
||||
// Verify no quality expectations or schema are set (semantics only)
|
||||
assertNull(dataContract.getQualityExpectations());
|
||||
assertNull(dataContract.getSchema());
|
||||
|
||||
// Verify FQN follows expected pattern
|
||||
String expectedFQN = chart.getFullyQualifiedName() + ".dataContract_" + create.getName();
|
||||
assertEquals(expectedFQN, dataContract.getFullyQualifiedName());
|
||||
|
||||
// Test the validate method and verify contract status
|
||||
DataContractResult validationResult = runValidate(dataContract);
|
||||
|
||||
// Verify the validation result
|
||||
assertNotNull(validationResult);
|
||||
assertNotNull(validationResult.getContractExecutionStatus());
|
||||
|
||||
// Verify semantics validation was performed
|
||||
assertNotNull(validationResult.getSemanticsValidation());
|
||||
assertEquals(2, validationResult.getSemanticsValidation().getTotal().intValue());
|
||||
|
||||
assertTrue(validationResult.getSemanticsValidation().getTotal() > 0);
|
||||
|
||||
// Verify no schema or quality validation was performed (semantics only)
|
||||
assertNull(validationResult.getSchemaValidation());
|
||||
assertNull(validationResult.getQualityValidation());
|
||||
|
||||
// Retrieve the contract and verify the latest result is stored
|
||||
DataContract retrievedContract = getDataContract(dataContract.getId(), "");
|
||||
assertNotNull(retrievedContract.getLatestResult());
|
||||
assertEquals(validationResult.getId(), retrievedContract.getLatestResult().getResultId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void testDashboardEntityConstraints(TestInfo test) throws IOException {
|
||||
@ -3720,8 +3863,20 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void testTopicEntityConstraints(TestInfo test) throws IOException {
|
||||
// Define message schema fields that match the data contract columns
|
||||
List<Field> messageSchemaFields =
|
||||
List.of(
|
||||
new Field()
|
||||
.withName("messageId")
|
||||
.withDisplayName("Message ID")
|
||||
.withDataType(FieldDataType.STRING),
|
||||
new Field()
|
||||
.withName("eventType")
|
||||
.withDisplayName("Event Type")
|
||||
.withDataType(FieldDataType.STRING));
|
||||
|
||||
// Test 1: Topic with schema should succeed (topics support schema validation)
|
||||
Topic schemaTopic = createUniqueTopic(test.getDisplayName() + "_schema");
|
||||
Topic schemaTopic = createUniqueTopic(test.getDisplayName() + "_schema", messageSchemaFields);
|
||||
|
||||
List<Column> columns =
|
||||
List.of(
|
||||
@ -3748,10 +3903,15 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
assertEquals("messageId", schemaContract.getSchema().get(0).getName());
|
||||
assertEquals("eventType", schemaContract.getSchema().get(1).getName());
|
||||
|
||||
// Test validation works for schema
|
||||
// Test validation works for schema - should pass when schema fields match
|
||||
DataContractResult schemaValidationResult = runValidate(schemaContract);
|
||||
assertNotNull(schemaValidationResult);
|
||||
assertEquals(
|
||||
ContractExecutionStatus.Success, schemaValidationResult.getContractExecutionStatus());
|
||||
assertNotNull(schemaValidationResult.getSchemaValidation());
|
||||
assertEquals(0, schemaValidationResult.getSchemaValidation().getFailed().intValue());
|
||||
assertEquals(2, schemaValidationResult.getSchemaValidation().getPassed().intValue());
|
||||
assertEquals(2, schemaValidationResult.getSchemaValidation().getTotal().intValue());
|
||||
|
||||
// Test 2: Topic with semantics should succeed
|
||||
Topic semanticsTopic = createUniqueTopic(test.getDisplayName() + "_semantics");
|
||||
@ -3778,7 +3938,7 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
assertNotNull(semanticsValidationResult.getSemanticsValidation());
|
||||
|
||||
// Test 3: Topic with both schema and semantics should succeed
|
||||
Topic bothTopic = createUniqueTopic(test.getDisplayName() + "_both");
|
||||
Topic bothTopic = createUniqueTopic(test.getDisplayName() + "_both", messageSchemaFields);
|
||||
|
||||
CreateDataContract createBoth =
|
||||
createDataContractRequestForEntity(test.getDisplayName() + "_both", bothTopic)
|
||||
@ -3826,11 +3986,126 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
assertEquals(schemaTopic.getName(), schemaContract.getEntity().getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void testTopicSchemaValidationFailure(TestInfo test) throws IOException {
|
||||
// Create topic message schema fields that match what the data contract will expect initially
|
||||
List<Field> messageSchemaFields =
|
||||
List.of(
|
||||
new Field()
|
||||
.withName("messageId")
|
||||
.withDisplayName("Message ID")
|
||||
.withDataType(FieldDataType.STRING),
|
||||
new Field()
|
||||
.withName("eventType")
|
||||
.withDisplayName("Event Type")
|
||||
.withDataType(FieldDataType.STRING));
|
||||
|
||||
Topic schemaTopic =
|
||||
createUniqueTopic(test.getDisplayName() + "_schema_fail", messageSchemaFields);
|
||||
|
||||
// Create data contract with schema fields that match the topic's message schema
|
||||
List<Column> columns =
|
||||
List.of(
|
||||
new Column()
|
||||
.withName("messageId")
|
||||
.withDescription("Message ID")
|
||||
.withDataType(ColumnDataType.STRING),
|
||||
new Column()
|
||||
.withName("eventType")
|
||||
.withDescription("Event Type")
|
||||
.withDataType(ColumnDataType.STRING));
|
||||
|
||||
CreateDataContract createWithSchema =
|
||||
createDataContractRequestForEntity(test.getDisplayName() + "_schema_fail", schemaTopic)
|
||||
.withSchema(columns);
|
||||
|
||||
// Create data contract with schema that matches the topic's message schema
|
||||
DataContract schemaContract = createDataContract(createWithSchema);
|
||||
assertNotNull(schemaContract);
|
||||
assertNotNull(schemaContract.getSchema());
|
||||
assertEquals(2, schemaContract.getSchema().size());
|
||||
|
||||
// First, validate the contract when the schema is still valid - should pass
|
||||
DataContractResult initialResult = runValidate(schemaContract);
|
||||
assertNotNull(initialResult);
|
||||
assertEquals(ContractExecutionStatus.Success, initialResult.getContractExecutionStatus());
|
||||
assertEquals(0, initialResult.getSchemaValidation().getFailed().intValue());
|
||||
assertEquals(2, initialResult.getSchemaValidation().getPassed().intValue());
|
||||
|
||||
// Now let's "break" the topic by removing one of the fields that the contract expects
|
||||
// We'll simulate this by updating the topic's message schema to have fewer fields
|
||||
String originalTopicJson = JsonUtils.pojoToJson(schemaTopic);
|
||||
|
||||
// Remove the "eventType" field from the topic's message schema
|
||||
List<Field> updatedFields = new ArrayList<>();
|
||||
for (Field field : schemaTopic.getMessageSchema().getSchemaFields()) {
|
||||
if (!"eventType".equals(field.getName())) {
|
||||
updatedFields.add(field);
|
||||
}
|
||||
}
|
||||
MessageSchema updatedMessageSchema =
|
||||
schemaTopic.getMessageSchema().withSchemaFields(updatedFields);
|
||||
schemaTopic.setMessageSchema(updatedMessageSchema);
|
||||
|
||||
// Patch the topic to remove the eventType field using TopicResourceTest
|
||||
TopicResourceTest topicResourceTest = new TopicResourceTest();
|
||||
Topic patchedTopic =
|
||||
topicResourceTest.patchEntity(
|
||||
schemaTopic.getId(), originalTopicJson, schemaTopic, ADMIN_AUTH_HEADERS);
|
||||
|
||||
// Verify the eventType field was removed from message schema
|
||||
assertEquals(1, patchedTopic.getMessageSchema().getSchemaFields().size());
|
||||
assertNull(
|
||||
patchedTopic.getMessageSchema().getSchemaFields().stream()
|
||||
.filter(field -> "eventType".equals(field.getName()))
|
||||
.findFirst()
|
||||
.orElse(null));
|
||||
|
||||
// Now validate the data contract - it should fail schema validation
|
||||
DataContractResult result = runValidate(schemaContract);
|
||||
|
||||
// Verify the validation result shows failure due to schema validation
|
||||
assertNotNull(result);
|
||||
assertEquals(ContractExecutionStatus.Failed, result.getContractExecutionStatus());
|
||||
|
||||
// Verify schema validation details
|
||||
assertNotNull(result.getSchemaValidation());
|
||||
assertEquals(
|
||||
1, result.getSchemaValidation().getFailed().intValue()); // 1 field failed (eventType)
|
||||
assertEquals(
|
||||
1, result.getSchemaValidation().getPassed().intValue()); // 1 field passed (messageId)
|
||||
assertEquals(
|
||||
2, result.getSchemaValidation().getTotal().intValue()); // 2 total fields in contract
|
||||
|
||||
// Verify the failed field is the eventType field
|
||||
assertNotNull(result.getSchemaValidation().getFailedFields());
|
||||
assertEquals(1, result.getSchemaValidation().getFailedFields().size());
|
||||
assertEquals("eventType", result.getSchemaValidation().getFailedFields().get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void testApiEndpointEntityConstraints(TestInfo test) throws IOException {
|
||||
// Test 1: API Endpoint with schema should succeed (apiEndpoint supports schema validation)
|
||||
APIEndpoint schemaApiEndpoint = createUniqueApiEndpoint(test.getDisplayName() + "_schema");
|
||||
// Create schema fields for the API endpoint that match what the data contract expects
|
||||
List<Field> requestSchemaFields =
|
||||
List.of(
|
||||
new Field()
|
||||
.withName("requestId")
|
||||
.withDataType(FieldDataType.STRING)
|
||||
.withDescription("Request ID"));
|
||||
List<Field> responseSchemaFields =
|
||||
List.of(
|
||||
new Field()
|
||||
.withName("responseCode")
|
||||
.withDataType(FieldDataType.INT)
|
||||
.withDescription("Response Code"));
|
||||
|
||||
APIEndpoint schemaApiEndpoint =
|
||||
createUniqueApiEndpoint(
|
||||
test.getDisplayName() + "_schema", requestSchemaFields, responseSchemaFields);
|
||||
|
||||
List<Column> columns =
|
||||
List.of(
|
||||
new Column()
|
||||
@ -3852,10 +4127,15 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
// Verify the schema fields match what we provided
|
||||
assertEquals("requestId", schemaContract.getSchema().get(0).getName());
|
||||
assertEquals("responseCode", schemaContract.getSchema().get(1).getName());
|
||||
// Test validation works for schema
|
||||
// Test validation works for schema - should pass when schema fields match
|
||||
DataContractResult schemaValidationResult = runValidate(schemaContract);
|
||||
assertNotNull(schemaValidationResult);
|
||||
assertEquals(
|
||||
ContractExecutionStatus.Success, schemaValidationResult.getContractExecutionStatus());
|
||||
assertNotNull(schemaValidationResult.getSchemaValidation());
|
||||
assertEquals(0, schemaValidationResult.getSchemaValidation().getFailed().intValue());
|
||||
assertEquals(2, schemaValidationResult.getSchemaValidation().getPassed().intValue());
|
||||
assertEquals(2, schemaValidationResult.getSchemaValidation().getTotal().intValue());
|
||||
|
||||
// Test 2: API Endpoint with semantics should succeed
|
||||
APIEndpoint semanticsApiEndpoint =
|
||||
@ -3880,7 +4160,9 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
assertNotNull(semanticsValidationResult.getSemanticsValidation());
|
||||
|
||||
// Test 3: API Endpoint with both schema and semantics should succeed
|
||||
APIEndpoint bothApiEndpoint = createUniqueApiEndpoint(test.getDisplayName() + "_both");
|
||||
APIEndpoint bothApiEndpoint =
|
||||
createUniqueApiEndpoint(
|
||||
test.getDisplayName() + "_both", requestSchemaFields, responseSchemaFields);
|
||||
CreateDataContract createBoth =
|
||||
createDataContractRequestForEntity(test.getDisplayName() + "_both", bothApiEndpoint)
|
||||
.withSchema(columns)
|
||||
@ -3922,13 +4204,125 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
assertEquals(schemaApiEndpoint.getName(), schemaContract.getEntity().getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void testApiEndpointSchemaValidationFailure(TestInfo test) throws IOException {
|
||||
// Create schema fields for the API endpoint that match what the data contract expects initially
|
||||
List<Field> requestSchemaFields =
|
||||
List.of(
|
||||
new Field()
|
||||
.withName("requestId")
|
||||
.withDataType(FieldDataType.STRING)
|
||||
.withDescription("Request ID"));
|
||||
List<Field> responseSchemaFields =
|
||||
List.of(
|
||||
new Field()
|
||||
.withName("responseCode")
|
||||
.withDataType(FieldDataType.INT)
|
||||
.withDescription("Response Code"));
|
||||
|
||||
APIEndpoint schemaApiEndpoint =
|
||||
createUniqueApiEndpoint(
|
||||
test.getDisplayName() + "_schema", requestSchemaFields, responseSchemaFields);
|
||||
|
||||
List<Column> columns =
|
||||
List.of(
|
||||
new Column()
|
||||
.withName("requestId")
|
||||
.withDescription("Request ID")
|
||||
.withDataType(ColumnDataType.STRING),
|
||||
new Column()
|
||||
.withName("responseCode")
|
||||
.withDescription("Response Code")
|
||||
.withDataType(ColumnDataType.INT));
|
||||
|
||||
CreateDataContract createWithSchema =
|
||||
createDataContractRequestForEntity(test.getDisplayName() + "_schema", schemaApiEndpoint)
|
||||
.withSchema(columns);
|
||||
|
||||
// Create data contract with schema that matches the API endpoint
|
||||
DataContract schemaContract = createDataContract(createWithSchema);
|
||||
assertNotNull(schemaContract);
|
||||
assertNotNull(schemaContract.getSchema());
|
||||
assertEquals(2, schemaContract.getSchema().size());
|
||||
|
||||
// First, validate the contract when the schema is still valid - should pass
|
||||
DataContractResult initialResult = runValidate(schemaContract);
|
||||
assertNotNull(initialResult);
|
||||
assertEquals(ContractExecutionStatus.Success, initialResult.getContractExecutionStatus());
|
||||
assertEquals(0, initialResult.getSchemaValidation().getFailed().intValue());
|
||||
assertEquals(2, initialResult.getSchemaValidation().getPassed().intValue());
|
||||
|
||||
// Now let's "break" the API endpoint by removing one of the fields that the contract expects
|
||||
// We'll simulate this by updating the API endpoint to have fewer response fields
|
||||
String originalApiEndpointJson = JsonUtils.pojoToJson(schemaApiEndpoint);
|
||||
|
||||
// Remove the "responseCode" field from the response schema
|
||||
List<Field> updatedResponseFields = new ArrayList<>();
|
||||
for (Field field : schemaApiEndpoint.getResponseSchema().getSchemaFields()) {
|
||||
if (!"responseCode".equals(field.getName())) {
|
||||
updatedResponseFields.add(field);
|
||||
}
|
||||
}
|
||||
schemaApiEndpoint.getResponseSchema().setSchemaFields(updatedResponseFields);
|
||||
|
||||
// Patch the API endpoint to remove the responseCode field using APIEndpointResourceTest
|
||||
APIEndpointResourceTest apiEndpointResourceTest = new APIEndpointResourceTest();
|
||||
APIEndpoint patchedApiEndpoint =
|
||||
apiEndpointResourceTest.patchEntity(
|
||||
schemaApiEndpoint.getId(),
|
||||
originalApiEndpointJson,
|
||||
schemaApiEndpoint,
|
||||
ADMIN_AUTH_HEADERS);
|
||||
|
||||
// Verify the responseCode field was removed from response schema
|
||||
assertEquals(0, patchedApiEndpoint.getResponseSchema().getSchemaFields().size());
|
||||
assertNull(
|
||||
patchedApiEndpoint.getResponseSchema().getSchemaFields().stream()
|
||||
.filter(field -> "responseCode".equals(field.getName()))
|
||||
.findFirst()
|
||||
.orElse(null));
|
||||
|
||||
// Now validate the data contract - it should fail schema validation
|
||||
DataContractResult result = runValidate(schemaContract);
|
||||
|
||||
// Verify the validation result shows failure due to schema validation
|
||||
assertNotNull(result);
|
||||
assertEquals(ContractExecutionStatus.Failed, result.getContractExecutionStatus());
|
||||
|
||||
// Verify schema validation details
|
||||
assertNotNull(result.getSchemaValidation());
|
||||
assertEquals(
|
||||
1, result.getSchemaValidation().getFailed().intValue()); // 1 field failed (responseCode)
|
||||
assertEquals(
|
||||
1, result.getSchemaValidation().getPassed().intValue()); // 1 field passed (requestId)
|
||||
assertEquals(
|
||||
2, result.getSchemaValidation().getTotal().intValue()); // 2 total fields in contract
|
||||
|
||||
// Verify the failed field is the responseCode field
|
||||
assertNotNull(result.getSchemaValidation().getFailedFields());
|
||||
assertEquals(1, result.getSchemaValidation().getFailedFields().size());
|
||||
assertEquals("responseCode", result.getSchemaValidation().getFailedFields().get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void testDashboardDataModelEntityConstraints(TestInfo test) throws IOException {
|
||||
// Test 1: Dashboard Data Model with schema should succeed (dashboardDataModel supports schema
|
||||
// validation)
|
||||
DashboardDataModel schemaDataModel =
|
||||
createUniqueDashboardDataModel(test.getDisplayName() + "_schema");
|
||||
// Create columns that match what the data contract will expect
|
||||
List<org.openmetadata.schema.type.Column> dataModelColumns =
|
||||
List.of(
|
||||
new org.openmetadata.schema.type.Column()
|
||||
.withName("metricId")
|
||||
.withDescription("Metric ID")
|
||||
.withDataType(org.openmetadata.schema.type.ColumnDataType.STRING),
|
||||
new org.openmetadata.schema.type.Column()
|
||||
.withName("metricValue")
|
||||
.withDescription("Metric Value")
|
||||
.withDataType(org.openmetadata.schema.type.ColumnDataType.DOUBLE));
|
||||
|
||||
// Create corresponding data contract columns
|
||||
List<Column> columns =
|
||||
List.of(
|
||||
new Column()
|
||||
@ -3939,6 +4333,10 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
.withName("metricValue")
|
||||
.withDescription("Metric Value")
|
||||
.withDataType(ColumnDataType.DOUBLE));
|
||||
|
||||
DashboardDataModel schemaDataModel =
|
||||
createUniqueDashboardDataModel(test.getDisplayName() + "_schema", dataModelColumns);
|
||||
|
||||
CreateDataContract createWithSchema =
|
||||
createDataContractRequestForEntity(test.getDisplayName() + "_schema", schemaDataModel)
|
||||
.withSchema(columns);
|
||||
@ -3950,10 +4348,15 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
// Verify the schema fields match what we provided
|
||||
assertEquals("metricId", schemaContract.getSchema().get(0).getName());
|
||||
assertEquals("metricValue", schemaContract.getSchema().get(1).getName());
|
||||
// Test validation works for schema
|
||||
// Test validation works for schema - should pass when schema fields match
|
||||
DataContractResult schemaValidationResult = runValidate(schemaContract);
|
||||
assertNotNull(schemaValidationResult);
|
||||
assertEquals(
|
||||
ContractExecutionStatus.Success, schemaValidationResult.getContractExecutionStatus());
|
||||
assertNotNull(schemaValidationResult.getSchemaValidation());
|
||||
assertEquals(0, schemaValidationResult.getSchemaValidation().getFailed().intValue());
|
||||
assertEquals(2, schemaValidationResult.getSchemaValidation().getPassed().intValue());
|
||||
assertEquals(2, schemaValidationResult.getSchemaValidation().getTotal().intValue());
|
||||
|
||||
// Test 2: Dashboard Data Model with semantics should succeed
|
||||
DashboardDataModel semanticsDataModel =
|
||||
@ -3978,7 +4381,7 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
|
||||
// Test 3: Dashboard Data Model with both schema and semantics should succeed
|
||||
DashboardDataModel bothDataModel =
|
||||
createUniqueDashboardDataModel(test.getDisplayName() + "_both");
|
||||
createUniqueDashboardDataModel(test.getDisplayName() + "_both", dataModelColumns);
|
||||
CreateDataContract createBoth =
|
||||
createDataContractRequestForEntity(test.getDisplayName() + "_both", bothDataModel)
|
||||
.withSchema(columns)
|
||||
@ -4021,6 +4424,102 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
|
||||
assertEquals(schemaDataModel.getName(), schemaContract.getEntity().getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void testDashboardDataModelSchemaValidationFailure(TestInfo test) throws IOException {
|
||||
// Create columns that match what the data contract will expect initially
|
||||
List<org.openmetadata.schema.type.Column> dataModelColumns =
|
||||
List.of(
|
||||
new org.openmetadata.schema.type.Column()
|
||||
.withName("metricId")
|
||||
.withDescription("Metric ID")
|
||||
.withDataType(org.openmetadata.schema.type.ColumnDataType.STRING),
|
||||
new org.openmetadata.schema.type.Column()
|
||||
.withName("metricValue")
|
||||
.withDescription("Metric Value")
|
||||
.withDataType(org.openmetadata.schema.type.ColumnDataType.DOUBLE));
|
||||
|
||||
// Create corresponding data contract columns
|
||||
List<Column> columns =
|
||||
List.of(
|
||||
new Column()
|
||||
.withName("metricId")
|
||||
.withDescription("Metric ID")
|
||||
.withDataType(ColumnDataType.STRING),
|
||||
new Column()
|
||||
.withName("metricValue")
|
||||
.withDescription("Metric Value")
|
||||
.withDataType(ColumnDataType.DOUBLE));
|
||||
|
||||
DashboardDataModel schemaDataModel =
|
||||
createUniqueDashboardDataModel(test.getDisplayName() + "_schema", dataModelColumns);
|
||||
|
||||
CreateDataContract createWithSchema =
|
||||
createDataContractRequestForEntity(test.getDisplayName() + "_schema", schemaDataModel)
|
||||
.withSchema(columns);
|
||||
|
||||
// Create data contract with schema that matches the data model
|
||||
DataContract schemaContract = createDataContract(createWithSchema);
|
||||
assertNotNull(schemaContract);
|
||||
assertNotNull(schemaContract.getSchema());
|
||||
assertEquals(2, schemaContract.getSchema().size());
|
||||
|
||||
// First, validate the contract when the schema is still valid - should pass
|
||||
DataContractResult initialResult = runValidate(schemaContract);
|
||||
assertNotNull(initialResult);
|
||||
assertEquals(ContractExecutionStatus.Success, initialResult.getContractExecutionStatus());
|
||||
assertEquals(0, initialResult.getSchemaValidation().getFailed().intValue());
|
||||
assertEquals(2, initialResult.getSchemaValidation().getPassed().intValue());
|
||||
|
||||
// Now let's "break" the data model by removing one of the columns that the contract expects
|
||||
// We'll simulate this by updating the data model to have fewer columns
|
||||
String originalDataModelJson = JsonUtils.pojoToJson(schemaDataModel);
|
||||
|
||||
// Remove the "metricValue" column from the data model columns
|
||||
List<org.openmetadata.schema.type.Column> updatedColumns = new ArrayList<>();
|
||||
for (org.openmetadata.schema.type.Column col : schemaDataModel.getColumns()) {
|
||||
if (!"metricValue".equals(col.getName())) {
|
||||
updatedColumns.add(col);
|
||||
}
|
||||
}
|
||||
schemaDataModel.setColumns(updatedColumns);
|
||||
|
||||
// Patch the data model to remove the metricValue column using DashboardDataModelResourceTest
|
||||
DashboardDataModelResourceTest dataModelResourceTest = new DashboardDataModelResourceTest();
|
||||
DashboardDataModel patchedDataModel =
|
||||
dataModelResourceTest.patchEntity(
|
||||
schemaDataModel.getId(), originalDataModelJson, schemaDataModel, ADMIN_AUTH_HEADERS);
|
||||
|
||||
// Verify the metricValue column was removed
|
||||
assertEquals(1, patchedDataModel.getColumns().size());
|
||||
assertNull(
|
||||
patchedDataModel.getColumns().stream()
|
||||
.filter(col -> "metricValue".equals(col.getName()))
|
||||
.findFirst()
|
||||
.orElse(null));
|
||||
|
||||
// Now validate the data contract - it should fail schema validation
|
||||
DataContractResult result = runValidate(schemaContract);
|
||||
|
||||
// Verify the validation result shows failure due to schema validation
|
||||
assertNotNull(result);
|
||||
assertEquals(ContractExecutionStatus.Failed, result.getContractExecutionStatus());
|
||||
|
||||
// Verify schema validation details
|
||||
assertNotNull(result.getSchemaValidation());
|
||||
assertEquals(
|
||||
1, result.getSchemaValidation().getFailed().intValue()); // 1 field failed (metricValue)
|
||||
assertEquals(
|
||||
1, result.getSchemaValidation().getPassed().intValue()); // 1 field passed (metricId)
|
||||
assertEquals(
|
||||
2, result.getSchemaValidation().getTotal().intValue()); // 2 total fields in contract
|
||||
|
||||
// Verify the failed field is the metricValue column
|
||||
assertNotNull(result.getSchemaValidation().getFailedFields());
|
||||
assertEquals(1, result.getSchemaValidation().getFailedFields().size());
|
||||
assertEquals("metricValue", result.getSchemaValidation().getFailedFields().get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void testTableEntityConstraints(TestInfo test) throws IOException {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user