diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index ac11bcf4cc8..f584a94d9b8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -34,9 +34,9 @@ import org.openmetadata.service.util.EntityUtil.Fields; public class DataContractRepository extends EntityRepository { private static final String DATA_CONTRACT_UPDATE_FIELDS = - "entity,owners,status,schema,qualityExpectations,contractUpdates,semantics"; + "entity,owners,reviewers,status,schema,qualityExpectations,contractUpdates,semantics,scheduleConfig"; private static final String DATA_CONTRACT_PATCH_FIELDS = - "entity,owners,status,schema,qualityExpectations,contractUpdates,semantics"; + "entity,owners,reviewers,status,schema,qualityExpectations,contractUpdates,semantics,scheduleConfig"; public DataContractRepository() { super( @@ -72,6 +72,9 @@ public class DataContractRepository extends EntityRepository { if (dataContract.getOwners() != null) { dataContract.setOwners(EntityUtil.populateEntityReferences(dataContract.getOwners())); } + if (dataContract.getReviewers() != null) { + dataContract.setReviewers(EntityUtil.populateEntityReferences(dataContract.getReviewers())); + } } private void validateSchemaFieldsAgainstEntity( @@ -105,12 +108,12 @@ public class DataContractRepository extends EntityRepository { Set tableColumnNames = table.getColumns().stream().map(Column::getName).collect(Collectors.toSet()); - for (org.openmetadata.schema.type.Field field : dataContract.getSchema()) { - if (!tableColumnNames.contains(field.getName())) { + for (org.openmetadata.schema.type.Column column : dataContract.getSchema()) { + if (!tableColumnNames.contains(column.getName())) { throw BadRequestException.of( String.format( "Field '%s' specified in the data contract does not exist in table '%s'", - field.getName(), table.getName())); + column.getName(), table.getName())); } } } @@ -127,12 +130,12 @@ public class DataContractRepository extends EntityRepository { Set topicFieldNames = extractFieldNames(topic.getMessageSchema().getSchemaFields()); - for (org.openmetadata.schema.type.Field field : dataContract.getSchema()) { - if (!topicFieldNames.contains(field.getName())) { + for (org.openmetadata.schema.type.Column column : dataContract.getSchema()) { + if (!topicFieldNames.contains(column.getName())) { throw BadRequestException.of( String.format( "Field '%s' specified in the data contract does not exist in topic '%s'", - field.getName(), topic.getName())); + column.getName(), topic.getName())); } } } @@ -167,6 +170,7 @@ public class DataContractRepository extends EntityRepository { Relationship.HAS); storeOwners(dataContract, dataContract.getOwners()); + storeReviewers(dataContract, dataContract.getReviewers()); } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractMapper.java index 60677dad201..7339634f25f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractMapper.java @@ -39,6 +39,7 @@ public class DataContractMapper { .withSemantics(create.getSemantics()) .withQualityExpectations(create.getQualityExpectations()) .withOwners(create.getOwners()) + .withReviewers(create.getReviewers()) .withEffectiveFrom(create.getEffectiveFrom()) .withEffectiveUntil(create.getEffectiveUntil()) .withSourceUrl(create.getSourceUrl()) @@ -50,6 +51,7 @@ public class DataContractMapper { public static DataContract trimFields(DataContract dataContract, Include include) { dataContract.setOwners(EntityUtil.getEntityReferences(dataContract.getOwners(), include)); + dataContract.setReviewers(EntityUtil.getEntityReferences(dataContract.getReviewers(), include)); if (include.value().equals("entity") || include.value().equals("all")) { dataContract.setEntity(Entity.getEntityReference(dataContract.getEntity(), include)); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractResource.java index 10b2b56093c..43248f6236e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractResource.java @@ -44,21 +44,32 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import jakarta.ws.rs.core.UriInfo; +import java.util.List; import java.util.UUID; +import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.data.CreateDataContract; +import org.openmetadata.schema.api.data.CreateDataContractResult; import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.entity.data.DataContract; +import org.openmetadata.schema.entity.datacontract.DataContractResult; import org.openmetadata.schema.type.EntityHistory; import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.DataContractRepository; +import org.openmetadata.service.jdbi3.EntityTimeSeriesDAO; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.limits.Limits; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.ResourceContext; +import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.ResultList; +@Slf4j @Path("/v1/dataContracts") @Tag( name = "Data Contracts", @@ -68,12 +79,13 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "dataContracts") public class DataContractResource extends EntityResource { public static final String COLLECTION_PATH = "v1/dataContracts/"; - static final String FIELDS = "owners"; + static final String FIELDS = "owners,reviewers"; @Override public DataContract addHref(UriInfo uriInfo, DataContract dataContract) { super.addHref(uriInfo, dataContract); Entity.withHref(uriInfo, dataContract.getOwners()); + Entity.withHref(uriInfo, dataContract.getReviewers()); Entity.withHref(uriInfo, dataContract.getEntity()); return dataContract; } @@ -530,6 +542,314 @@ public class DataContractResource extends EntityResource listResults( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the data contract", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @Parameter(description = "Limit the number of results (1 to 10000, default = 10)") + @DefaultValue("10") + @QueryParam("limit") + @Min(0) + @Max(10000) + int limitParam, + @Parameter( + description = "Returns results after this timestamp", + schema = @Schema(type = "number")) + @QueryParam("startTs") + Long startTs, + @Parameter( + description = "Returns results before this timestamp", + schema = @Schema(type = "number")) + @QueryParam("endTs") + Long endTs) { + DataContract dataContract = repository.get(uriInfo, id, Fields.EMPTY_FIELDS); + OperationContext operationContext = + new OperationContext(Entity.DATA_CONTRACT, MetadataOperation.VIEW_BASIC); + ResourceContext resourceContext = + new ResourceContext<>(Entity.DATA_CONTRACT, id, null); + authorizer.authorize(securityContext, operationContext, resourceContext); + + EntityTimeSeriesDAO timeSeriesDAO = Entity.getCollectionDAO().entityExtensionTimeSeriesDao(); + List jsonResults = + timeSeriesDAO.listBetweenTimestampsByOrder( + dataContract.getFullyQualifiedName(), + "dataContract.dataContractResult", + startTs != null ? startTs : 0L, + endTs != null ? endTs : System.currentTimeMillis(), + EntityTimeSeriesDAO.OrderBy.DESC); + + List results = JsonUtils.readObjects(jsonResults, DataContractResult.class); + + // Apply limit + if (limitParam > 0 && results.size() > limitParam) { + results = results.subList(0, limitParam); + } + + return new ResultList<>( + results, String.valueOf(startTs), String.valueOf(endTs), results.size()); + } + + @GET + @Path("/{id}/results/latest") + @Operation( + operationId = "getLatestDataContractResult", + summary = "Get latest data contract result", + description = "Get the latest execution result for a data contract.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Latest data contract result", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = DataContractResult.class))), + @ApiResponse(responseCode = "404", description = "Data contract or result not found") + }) + public DataContractResult getLatestResult( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the data contract", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) + throws Exception { + DataContract dataContract = repository.get(uriInfo, id, Fields.EMPTY_FIELDS); + OperationContext operationContext = + new OperationContext(Entity.DATA_CONTRACT, MetadataOperation.VIEW_BASIC); + ResourceContext resourceContext = + new ResourceContext<>(Entity.DATA_CONTRACT, id, null); + authorizer.authorize(securityContext, operationContext, resourceContext); + + EntityTimeSeriesDAO timeSeriesDAO = Entity.getCollectionDAO().entityExtensionTimeSeriesDao(); + String jsonRecord = + timeSeriesDAO.getLatestExtension( + dataContract.getFullyQualifiedName(), "dataContract.dataContractResult"); + + return jsonRecord != null ? JsonUtils.readValue(jsonRecord, DataContractResult.class) : null; + } + + @GET + @Path("/{id}/results/{resultId}") + @Operation( + operationId = "getDataContractResult", + summary = "Get a data contract result by ID", + description = "Get a specific data contract execution result by its ID.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Data contract result", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = DataContractResult.class))), + @ApiResponse(responseCode = "404", description = "Data contract result not found") + }) + public DataContractResult getResult( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the data contract", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @Parameter(description = "Id of the data contract result", schema = @Schema(type = "UUID")) + @PathParam("resultId") + UUID resultId) + throws Exception { + DataContract dataContract = repository.get(uriInfo, id, Fields.EMPTY_FIELDS); + OperationContext operationContext = + new OperationContext(Entity.DATA_CONTRACT, MetadataOperation.VIEW_BASIC); + ResourceContext resourceContext = + new ResourceContext<>(Entity.DATA_CONTRACT, id, null); + authorizer.authorize(securityContext, operationContext, resourceContext); + + EntityTimeSeriesDAO timeSeriesDAO = Entity.getCollectionDAO().entityExtensionTimeSeriesDao(); + String jsonRecord = timeSeriesDAO.getById(resultId); + return JsonUtils.readValue(jsonRecord, DataContractResult.class); + } + + @POST + @Path("/{id}/results") + @Operation( + operationId = "createDataContractResult", + summary = "Create or update data contract result", + description = "Create a new data contract execution result.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Successfully created or updated the result", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = DataContractResult.class))) + }) + public Response createResult( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the data contract", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @Valid CreateDataContractResult create) { + DataContract dataContract = repository.get(uriInfo, id, Fields.EMPTY_FIELDS); + OperationContext operationContext = + new OperationContext(Entity.DATA_CONTRACT, MetadataOperation.EDIT_ALL); + ResourceContext resourceContext = + new ResourceContext<>(Entity.DATA_CONTRACT, id, null); + authorizer.authorize(securityContext, operationContext, resourceContext); + + DataContractResult result = getContractResult(dataContract, create); + + EntityTimeSeriesDAO timeSeriesDAO = Entity.getCollectionDAO().entityExtensionTimeSeriesDao(); + timeSeriesDAO.insert( + dataContract.getFullyQualifiedName(), + "dataContract.dataContractResult", + "dataContractResult", + JsonUtils.pojoToJson(result)); + + // Update latest result in data contract + updateLatestResult(dataContract, result); + + return Response.ok(result).build(); + } + + @DELETE + @Path("/{id}/results/{timestamp}") + @Operation( + operationId = "deleteDataContractResult", + summary = "Delete data contract result", + description = "Delete a data contract result at a specific timestamp.", + responses = { + @ApiResponse(responseCode = "200", description = "Successfully deleted the result") + }) + public Response deleteResult( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the data contract", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @Parameter( + description = "Timestamp of the result to delete", + schema = @Schema(type = "number")) + @PathParam("timestamp") + Long timestamp) + throws Exception { + DataContract dataContract = repository.get(uriInfo, id, Fields.EMPTY_FIELDS); + OperationContext operationContext = + new OperationContext(Entity.DATA_CONTRACT, MetadataOperation.DELETE); + ResourceContext resourceContext = + new ResourceContext<>(Entity.DATA_CONTRACT, id, null); + authorizer.authorize(securityContext, operationContext, resourceContext); + + EntityTimeSeriesDAO timeSeriesDAO = Entity.getCollectionDAO().entityExtensionTimeSeriesDao(); + timeSeriesDAO.deleteAtTimestamp( + dataContract.getFullyQualifiedName(), "dataContract.dataContractResult", timestamp); + return Response.ok().build(); + } + + @DELETE + @Path("/{id}/results/before/{timestamp}") + @Operation( + operationId = "deleteDataContractResultsBefore", + summary = "Delete data contract results before timestamp", + description = "Delete all data contract results before a specific timestamp.", + responses = { + @ApiResponse(responseCode = "200", description = "Successfully deleted the results") + }) + public Response deleteResultsBefore( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the data contract", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @Parameter( + description = "Delete results before this timestamp", + schema = @Schema(type = "number")) + @PathParam("timestamp") + Long timestamp) { + DataContract dataContract = repository.get(uriInfo, id, Fields.EMPTY_FIELDS); + OperationContext operationContext = + new OperationContext(Entity.DATA_CONTRACT, MetadataOperation.DELETE); + ResourceContext resourceContext = + new ResourceContext<>(Entity.DATA_CONTRACT, id, null); + authorizer.authorize(securityContext, operationContext, resourceContext); + + EntityTimeSeriesDAO timeSeriesDAO = Entity.getCollectionDAO().entityExtensionTimeSeriesDao(); + timeSeriesDAO.deleteBeforeTimestamp( + dataContract.getFullyQualifiedName(), "dataContract.dataContractResult", timestamp); + return Response.ok().build(); + } + + private DataContractResult getContractResult( + DataContract dataContract, CreateDataContractResult create) { + DataContractResult result = + new DataContractResult() + .withId(UUID.randomUUID()) + .withDataContractFQN(dataContract.getFullyQualifiedName()) + .withTimestamp(create.getTimestamp()) + .withContractExecutionStatus(create.getContractExecutionStatus()) + .withResult(create.getResult()) + .withExecutionTime(create.getExecutionTime()); + + if (create.getSchemaValidation() != null) { + result.withSchemaValidation(create.getSchemaValidation()); + } + + if (create.getSemanticsValidation() != null) { + result.withSemanticsValidation(create.getSemanticsValidation()); + } + + if (create.getQualityValidation() != null) { + result.withQualityValidation(create.getQualityValidation()); + } + + if (create.getSlaValidation() != null) { + result.withSlaValidation(create.getSlaValidation()); + } + + if (create.getIncidentId() != null) { + result.withIncidentId(create.getIncidentId()); + } + + return result; + } + + private void updateLatestResult(DataContract dataContract, DataContractResult result) { + try { + jakarta.json.JsonPatchBuilder patchBuilder = jakarta.json.Json.createPatchBuilder(); + + jakarta.json.JsonObjectBuilder latestResultBuilder = + jakarta.json.Json.createObjectBuilder() + .add("timestamp", result.getTimestamp()) + .add("status", result.getContractExecutionStatus().value()) + .add("message", result.getResult() != null ? result.getResult() : "") + .add("resultId", result.getId().toString()); + patchBuilder.add("/latestResult", latestResultBuilder.build()); + JsonPatch patch = patchBuilder.build(); + repository.patch(null, dataContract.getId(), null, patch); + } catch (Exception e) { + LOG.error( + "Failed to update latest result for data contract {}", + dataContract.getFullyQualifiedName(), + e); + } + } + public static class DataContractList extends ResultList { @SuppressWarnings("unused") public DataContractList() { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index 33eedf26ad1..dd3f0358c67 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -94,7 +94,7 @@ import org.openmetadata.service.util.ResultList; public class TableResource extends EntityResource { private final TableMapper mapper = new TableMapper(); public static final String COLLECTION_PATH = "v1/tables/"; - static final String FIELDS = + public static final String FIELDS = "tableConstraints,tablePartition,usageSummary,owners,customMetrics,columns," + "tags,followers,joins,schemaDefinition,dataModel,extension,testSuite,domain,dataProducts,lifeCycle,sourceHash"; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java index 78ee0d80c57..6bdaa0b18aa 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java @@ -16,6 +16,7 @@ package org.openmetadata.service.resources.data; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; import static org.openmetadata.service.util.TestUtils.TEST_AUTH_HEADERS; import static org.openmetadata.service.util.TestUtils.assertResponse; @@ -41,6 +42,7 @@ import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; import org.openmetadata.schema.api.data.CreateDataContract; +import org.openmetadata.schema.api.data.CreateDataContractResult; import org.openmetadata.schema.api.data.CreateDatabase; import org.openmetadata.schema.api.data.CreateDatabaseSchema; import org.openmetadata.schema.api.data.CreateTable; @@ -51,17 +53,21 @@ import org.openmetadata.schema.entity.data.DataContract; import org.openmetadata.schema.entity.data.Database; import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.datacontract.DataContractResult; import org.openmetadata.schema.entity.services.DatabaseService; +import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.services.connections.database.MysqlConnection; +import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.ColumnDataType; +import org.openmetadata.schema.type.ContractExecutionStatus; import org.openmetadata.schema.type.ContractStatus; import org.openmetadata.schema.type.EntityReference; -import org.openmetadata.schema.type.Field; -import org.openmetadata.schema.type.FieldDataType; import org.openmetadata.schema.type.QualityExpectation; import org.openmetadata.schema.type.SemanticsRule; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.OpenMetadataApplicationTest; import org.openmetadata.service.security.SecurityUtil; +import org.openmetadata.service.util.ResultList; import org.openmetadata.service.util.TestUtils; @Slf4j @@ -178,7 +184,7 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { long counter = tableCounter.incrementAndGet(); long timestamp = System.nanoTime(); String uniqueId = UUID.randomUUID().toString().replace("-", ""); - long threadId = Thread.currentThread().getId(); + long threadId = Thread.currentThread().threadId(); String tableName = "dc_test_" @@ -235,7 +241,7 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { + "_" + System.nanoTime() + "_" - + Thread.currentThread().getId() + + Thread.currentThread().threadId() + "_" + tableCounter.incrementAndGet(); String contractName = "contract_" + name + "_" + uniqueSuffix; @@ -299,8 +305,6 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { response.readEntity(String.class); // Consume response } - // ===================== Permission Helper Methods ===================== - private DataContract createDataContractWithAuth( CreateDataContract create, Map authHeaders) throws IOException { WebTarget target = getCollection(); @@ -442,18 +446,18 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { String originalJson = JsonUtils.pojoToJson(created); // Add schema fields via patch - List schemaFields = new ArrayList<>(); - schemaFields.add( - new Field() + List columns = new ArrayList<>(); + columns.add( + new Column() .withName(C1) .withDescription("Updated ID field") - .withDataType(FieldDataType.INT)); - schemaFields.add( - new Field() + .withDataType(ColumnDataType.INT)); + columns.add( + new Column() .withName(C2) .withDescription("Updated name field") - .withDataType(FieldDataType.STRING)); - created.setSchema(schemaFields); + .withDataType(ColumnDataType.STRING)); + created.setSchema(columns); DataContract patched = patchDataContract(created.getId(), originalJson, created); @@ -507,13 +511,13 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { created.setStatus(ContractStatus.Active); created.setDescription("Updated contract description via patch"); - List schemaFields = new ArrayList<>(); - schemaFields.add( - new Field() + List columns = new ArrayList<>(); + columns.add( + new Column() .withName(C1) .withDescription("Patched ID field") - .withDataType(FieldDataType.INT)); - created.setSchema(schemaFields); + .withDataType(ColumnDataType.INT)); + created.setSchema(columns); DataContract patched = patchDataContract(created.getId(), originalJson, created); @@ -521,7 +525,7 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { assertEquals("Updated contract description via patch", patched.getDescription()); assertNotNull(patched.getSchema()); assertEquals(1, patched.getSchema().size()); - assertEquals("Patched ID field", patched.getSchema().get(0).getDescription()); + assertEquals("Patched ID field", patched.getSchema().getFirst().getDescription()); } @Test @@ -545,17 +549,20 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { Table table = createUniqueTable(test.getDisplayName()); // Add schema fields that match the table's columns - List schemaFields = new ArrayList<>(); - schemaFields.add( - new Field() + List columns = new ArrayList<>(); + columns.add( + new Column() .withName(C1) .withDescription("Unique identifier") - .withDataType(FieldDataType.INT)); - schemaFields.add( - new Field().withName(C2).withDescription("Name field").withDataType(FieldDataType.STRING)); + .withDataType(ColumnDataType.INT)); + columns.add( + new Column() + .withName(C2) + .withDescription("Name field") + .withDataType(ColumnDataType.STRING)); CreateDataContract create = - createDataContractRequest(test.getDisplayName(), table).withSchema(schemaFields); + createDataContractRequest(test.getDisplayName(), table).withSchema(columns); DataContract dataContract = createDataContract(create); @@ -585,7 +592,7 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { assertNotNull(dataContract.getQualityExpectations()); assertEquals(1, dataContract.getQualityExpectations().size()); - assertEquals("Completeness", dataContract.getQualityExpectations().get(0).getName()); + assertEquals("Completeness", dataContract.getQualityExpectations().getFirst().getName()); } @Test @@ -594,15 +601,15 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { Table table = createUniqueTable(test.getDisplayName()); // Create schema with field that doesn't exist in the table - List schemaFields = new ArrayList<>(); - schemaFields.add( - new Field() + List columns = new ArrayList<>(); + columns.add( + new Column() .withName("non_existent_field") .withDescription("This field doesn't exist") - .withDataType(FieldDataType.STRING)); + .withDataType(ColumnDataType.STRING)); CreateDataContract create = - createDataContractRequest(test.getDisplayName(), table).withSchema(schemaFields); + createDataContractRequest(test.getDisplayName(), table).withSchema(columns); // Should throw error for non-existent field assertResponseContains( @@ -643,7 +650,7 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { + "_" + System.nanoTime() + "_" - + Thread.currentThread().getId() + + Thread.currentThread().threadId() + "_" + tableCounter.incrementAndGet(); String contractName = "contract_" + test.getDisplayName() + "_" + uniqueSuffix; @@ -712,7 +719,7 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { assertEquals(ContractStatus.Active, dataContract.getStatus()); assertEquals(2, dataContract.getSchema().size()); assertEquals(1, dataContract.getQualityExpectations().size()); - assertEquals("EmailFormat", dataContract.getQualityExpectations().get(0).getName()); + assertEquals("EmailFormat", dataContract.getQualityExpectations().getFirst().getName()); } @Test @@ -770,25 +777,25 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { Table table = createUniqueTable(test.getDisplayName()); // Create schema with multiple fields that don't exist in the table - List schemaFields = new ArrayList<>(); - schemaFields.add( - new Field() + List columns = new ArrayList<>(); + columns.add( + new Column() .withName("invalid_field_1") .withDescription("First invalid field") - .withDataType(FieldDataType.STRING)); - schemaFields.add( - new Field() + .withDataType(ColumnDataType.STRING)); + columns.add( + new Column() .withName("invalid_field_2") .withDescription("Second invalid field") - .withDataType(FieldDataType.INT)); - schemaFields.add( - new Field() + .withDataType(ColumnDataType.INT)); + columns.add( + new Column() .withName(C1) // This one is valid .withDescription("Valid field") - .withDataType(FieldDataType.INT)); + .withDataType(ColumnDataType.INT)); CreateDataContract create = - createDataContractRequest(test.getDisplayName(), table).withSchema(schemaFields); + createDataContractRequest(test.getDisplayName(), table).withSchema(columns); // Should fail on the first invalid field encountered assertResponseContains( @@ -1013,4 +1020,308 @@ class DataContractResourceTest extends OpenMetadataApplicationTest { Status.FORBIDDEN, "Principal: CatalogPrincipal{name='test'} operations [Create] not allowed"); } + + // ===================== Reviewers Tests ===================== + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataContractWithReviewers(TestInfo test) throws IOException { + Table table = createUniqueTable(test.getDisplayName()); + + // Get admin user entity reference + Map authHeaders = SecurityUtil.authHeaders("admin@open-metadata.org"); + WebTarget userTarget = getResource("users").path("name").path("admin"); + Response response = SecurityUtil.addHeaders(userTarget, authHeaders).get(); + User adminUser = TestUtils.readResponse(response, User.class, Status.OK.getStatusCode()); + + // Create user references for reviewers using the full entity reference + List reviewers = new ArrayList<>(); + reviewers.add(adminUser.getEntityReference()); + + CreateDataContract create = + createDataContractRequest(test.getDisplayName(), table).withReviewers(reviewers); + + DataContract dataContract = createDataContract(create); + + // Get with reviewers field to verify they were set + DataContract retrieved = getDataContract(dataContract.getId(), "owners,reviewers"); + + assertNotNull(retrieved.getReviewers()); + assertEquals(1, retrieved.getReviewers().size()); + assertEquals("admin", retrieved.getReviewers().getFirst().getName()); + assertEquals("user", retrieved.getReviewers().getFirst().getType()); + assertNotNull(retrieved.getReviewers().getFirst().getId()); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testPatchDataContractAddReviewers(TestInfo test) throws IOException { + Table table = createUniqueTable(test.getDisplayName()); + CreateDataContract create = createDataContractRequest(test.getDisplayName(), table); + DataContract created = createDataContract(create); + + // Get admin user entity reference + Map authHeaders = SecurityUtil.authHeaders("admin@open-metadata.org"); + WebTarget userTarget = getResource("users").path("name").path("admin"); + Response response = SecurityUtil.addHeaders(userTarget, authHeaders).get(); + User adminUser = TestUtils.readResponse(response, User.class, Status.OK.getStatusCode()); + + // Get the full data contract with all fields + created = getDataContract(created.getId(), "reviewers"); + String originalJson = JsonUtils.pojoToJson(created); + + // Add reviewers via patch + List reviewers = new ArrayList<>(); + reviewers.add(adminUser.getEntityReference()); + created.setReviewers(reviewers); + + DataContract patched = patchDataContract(created.getId(), originalJson, created); + + assertNotNull(patched.getReviewers()); + assertEquals(1, patched.getReviewers().size()); + assertEquals("admin", patched.getReviewers().get(0).getName()); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testPatchDataContractRemoveReviewers(TestInfo test) throws IOException { + Table table = createUniqueTable(test.getDisplayName()); + + // Get admin user entity reference + Map authHeaders = SecurityUtil.authHeaders("admin@open-metadata.org"); + WebTarget userTarget = getResource("users").path("name").path("admin"); + Response response = SecurityUtil.addHeaders(userTarget, authHeaders).get(); + User adminUser = TestUtils.readResponse(response, User.class, Status.OK.getStatusCode()); + + // Create with reviewers + List initialReviewers = new ArrayList<>(); + initialReviewers.add(adminUser.getEntityReference()); + + CreateDataContract create = + createDataContractRequest(test.getDisplayName(), table).withReviewers(initialReviewers); + DataContract created = createDataContract(create); + + // Get full data contract with reviewers + created = getDataContract(created.getId(), "owners,reviewers"); + assertNotNull(created.getReviewers()); + assertEquals(1, created.getReviewers().size()); + + String originalJson = JsonUtils.pojoToJson(created); + + // Remove reviewers via patch + created.setReviewers(new ArrayList<>()); + + DataContract patched = patchDataContract(created.getId(), originalJson, created); + + assertNotNull(patched.getReviewers()); + assertEquals(0, patched.getReviewers().size()); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testPatchDataContractUpdateReviewers(TestInfo test) throws IOException { + Table table = createUniqueTable(test.getDisplayName()); + + // Get user entity references + Map authHeaders = SecurityUtil.authHeaders("admin@open-metadata.org"); + WebTarget userTarget = getResource("users").path("name").path("admin"); + Response response = SecurityUtil.addHeaders(userTarget, authHeaders).get(); + User adminUser = TestUtils.readResponse(response, User.class, Status.OK.getStatusCode()); + + userTarget = getResource("users").path("name").path("test"); + response = SecurityUtil.addHeaders(userTarget, authHeaders).get(); + User testUser = TestUtils.readResponse(response, User.class, Status.OK.getStatusCode()); + + // Create with one reviewer + List initialReviewers = new ArrayList<>(); + initialReviewers.add(adminUser.getEntityReference()); + + CreateDataContract create = + createDataContractRequest(test.getDisplayName(), table).withReviewers(initialReviewers); + DataContract created = createDataContract(create); + + // Get full data contract + created = getDataContract(created.getId(), "reviewers"); + String originalJson = JsonUtils.pojoToJson(created); + + // Update to different reviewers (test user) + List newReviewers = new ArrayList<>(); + newReviewers.add(testUser.getEntityReference()); + created.setReviewers(newReviewers); + + DataContract patched = patchDataContract(created.getId(), originalJson, created); + + assertNotNull(patched.getReviewers()); + assertEquals(1, patched.getReviewers().size()); + assertEquals("test", patched.getReviewers().get(0).getName()); + } + + // ===================== Data Contract Results Tests ===================== + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testCreateDataContractResult(TestInfo test) throws IOException { + Table table = createUniqueTable(test.getDisplayName()); + CreateDataContract create = createDataContractRequest(test.getDisplayName(), table); + DataContract dataContract = createDataContract(create); + + // Create a contract result + CreateDataContractResult createResult = + new CreateDataContractResult() + .withDataContractFQN(dataContract.getFullyQualifiedName()) + .withTimestamp(System.currentTimeMillis()) + .withContractExecutionStatus(ContractExecutionStatus.Success) + .withResult("All validations passed") + .withExecutionTime(1234L); + + WebTarget resultsTarget = getResource(dataContract.getId()).path("/results"); + Response response = + SecurityUtil.addHeaders(resultsTarget, ADMIN_AUTH_HEADERS).post(Entity.json(createResult)); + DataContractResult result = + TestUtils.readResponse(response, DataContractResult.class, Status.OK.getStatusCode()); + + assertNotNull(result); + assertNotNull(result.getId()); + assertEquals(dataContract.getFullyQualifiedName(), result.getDataContractFQN()); + assertEquals(ContractExecutionStatus.Success, result.getContractExecutionStatus()); + assertEquals("All validations passed", result.getResult()); + assertEquals(1234L, result.getExecutionTime()); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testListDataContractResults(TestInfo test) throws Exception { + Table table = createUniqueTable(test.getDisplayName()); + CreateDataContract create = createDataContractRequest(test.getDisplayName(), table); + DataContract dataContract = createDataContract(create); + + // Create multiple contract results with different dates + String dateStr = "2024-01-"; + for (int i = 1; i <= 5; i++) { + CreateDataContractResult createResult = + new CreateDataContractResult() + .withDataContractFQN(dataContract.getFullyQualifiedName()) + .withTimestamp(TestUtils.dateToTimestamp(dateStr + String.format("%02d", i))) + .withContractExecutionStatus( + i % 2 == 0 ? ContractExecutionStatus.Success : ContractExecutionStatus.Failed) + .withResult("Result " + i) + .withExecutionTime(1000L + i); + + WebTarget resultsTarget = getResource(dataContract.getId()).path("/results"); + Response response = + SecurityUtil.addHeaders(resultsTarget, ADMIN_AUTH_HEADERS) + .post(Entity.json(createResult)); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + response.readEntity(String.class); // Consume response + } + + // List results + WebTarget listTarget = getResource(dataContract.getId()).path("/results"); + Response listResponse = SecurityUtil.addHeaders(listTarget, ADMIN_AUTH_HEADERS).get(); + String jsonResponse = + TestUtils.readResponse(listResponse, String.class, Status.OK.getStatusCode()); + ResultList results = + JsonUtils.readValue( + jsonResponse, + new com.fasterxml.jackson.core.type.TypeReference>() {}); + + assertNotNull(results); + assertEquals(5, results.getData().size()); + + // Verify results are in descending order by timestamp (newest first) + for (int i = 0; i < results.getData().size() - 1; i++) { + assertTrue( + results.getData().get(i).getTimestamp() >= results.getData().get(i + 1).getTimestamp()); + } + + // Verify the newest result is from January 5th + assertEquals("Result 5", results.getData().get(0).getResult()); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testGetLatestDataContractResult(TestInfo test) throws Exception { + Table table = createUniqueTable(test.getDisplayName()); + CreateDataContract create = createDataContractRequest(test.getDisplayName(), table); + DataContract dataContract = createDataContract(create); + + // Create multiple results with different dates + String[] dates = {"2024-01-01", "2024-01-02", "2024-01-03"}; + for (int i = 0; i < dates.length; i++) { + CreateDataContractResult createResult = + new CreateDataContractResult() + .withDataContractFQN(dataContract.getFullyQualifiedName()) + .withTimestamp(TestUtils.dateToTimestamp(dates[i])) + .withContractExecutionStatus(ContractExecutionStatus.Success) + .withResult("Result " + i) + .withExecutionTime(1000L); + + WebTarget resultsTarget = getResource(dataContract.getId()).path("/results"); + Response response = + SecurityUtil.addHeaders(resultsTarget, ADMIN_AUTH_HEADERS) + .post(Entity.json(createResult)); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + response.readEntity(String.class); // Consume response + } + + // Get latest result + WebTarget latestTarget = getResource(dataContract.getId()).path("/results/latest"); + Response latestResponse = SecurityUtil.addHeaders(latestTarget, ADMIN_AUTH_HEADERS).get(); + DataContractResult latest = + TestUtils.readResponse(latestResponse, DataContractResult.class, Status.OK.getStatusCode()); + + assertNotNull(latest); + assertEquals("Result 2", latest.getResult()); // Latest is from January 3rd (index 2) + assertEquals(TestUtils.dateToTimestamp("2024-01-03"), latest.getTimestamp()); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDeleteDataContractResults(TestInfo test) throws Exception { + Table table = createUniqueTable(test.getDisplayName()); + CreateDataContract create = createDataContractRequest(test.getDisplayName(), table); + DataContract dataContract = createDataContract(create); + + // Create a result with a specific date + long timestamp = TestUtils.dateToTimestamp("2024-01-15"); + CreateDataContractResult createResult = + new CreateDataContractResult() + .withDataContractFQN(dataContract.getFullyQualifiedName()) + .withTimestamp(timestamp) + .withContractExecutionStatus(ContractExecutionStatus.Success) + .withResult("Test result") + .withExecutionTime(1000L); + + WebTarget resultsTarget = getResource(dataContract.getId()).path("/results"); + Response createResponse = + SecurityUtil.addHeaders(resultsTarget, ADMIN_AUTH_HEADERS).post(Entity.json(createResult)); + assertEquals(Status.OK.getStatusCode(), createResponse.getStatus()); + createResponse.readEntity(String.class); // Consume response + + // Verify result exists + WebTarget listTarget = getResource(dataContract.getId()).path("/results"); + Response listResponse = SecurityUtil.addHeaders(listTarget, ADMIN_AUTH_HEADERS).get(); + String jsonResponse = + TestUtils.readResponse(listResponse, String.class, Status.OK.getStatusCode()); + ResultList results = + JsonUtils.readValue( + jsonResponse, + new com.fasterxml.jackson.core.type.TypeReference>() {}); + assertEquals(1, results.getData().size()); + + // Delete the result + WebTarget deleteTarget = getResource(dataContract.getId()).path("/results/" + timestamp); + Response deleteResponse = SecurityUtil.addHeaders(deleteTarget, ADMIN_AUTH_HEADERS).delete(); + assertEquals(Status.OK.getStatusCode(), deleteResponse.getStatus()); + + // Verify result is deleted + Response listResponse2 = SecurityUtil.addHeaders(listTarget, ADMIN_AUTH_HEADERS).get(); + String jsonResponse2 = + TestUtils.readResponse(listResponse2, String.class, Status.OK.getStatusCode()); + ResultList results2 = + JsonUtils.readValue( + jsonResponse2, + new com.fasterxml.jackson.core.type.TypeReference>() {}); + assertEquals(0, results2.getData().size()); + } } diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createDataContract.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createDataContract.json index ccaff090545..b27c29e0faf 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createDataContract.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDataContract.json @@ -33,7 +33,7 @@ "description": "Schema definition for the data contract.", "type": "array", "items": { - "$ref": "../../entity/data/dataContract.json#/definitions/schemaField" + "$ref": "../../entity/data/table.json#/definitions/column" }, "default": null }, @@ -58,6 +58,11 @@ "$ref": "../../type/entityReferenceList.json", "default": null }, + "reviewers": { + "description": "User references of the reviewers for this data contract.", + "$ref": "../../type/entityReferenceList.json", + "default": null + }, "effectiveFrom": { "description": "Date from which this data contract is effective.", "$ref": "../../type/basic.json#/definitions/dateTime", @@ -71,6 +76,10 @@ "sourceUrl": { "description": "Source URL of the data contract.", "$ref": "../../type/basic.json#/definitions/sourceUrl" + }, + "scheduleConfig": { + "description": "Configuration for scheduling data contract validation checks.", + "$ref": "../../entity/data/dataContract.json#/properties/scheduleConfig" } }, "required": [ diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createDataContractResult.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createDataContractResult.json new file mode 100644 index 00000000000..59c9422f6ee --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDataContractResult.json @@ -0,0 +1,56 @@ +{ + "$id": "https://open-metadata.org/schema/api/data/createDataContractResult.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateDataContractResult", + "description": "Create data contract result API request.", + "type": "object", + "javaType": "org.openmetadata.schema.api.data.CreateDataContractResult", + "properties": { + "dataContractFQN": { + "description": "Fully qualified name of the data contract.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "timestamp": { + "description": "Timestamp when the data contract was executed.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "contractExecutionStatus": { + "description": "Overall status of the contract execution.", + "$ref": "../../type/contractExecutionStatus.json" + }, + "result": { + "description": "Detailed result of the data contract execution.", + "type": "string" + }, + "schemaValidation": { + "description": "Schema validation details.", + "$ref": "../../entity/datacontract/schemaValidation.json", + "default": null + }, + "semanticsValidation": { + "description": "Semantics validation details.", + "$ref": "../../entity/datacontract/semanticsValidation.json", + "default": null + }, + "qualityValidation": { + "description": "Quality expectations validation details.", + "$ref": "../../entity/datacontract/qualityValidation.json", + "default": null + }, + "slaValidation": { + "description": "SLA validation details.", + "$ref": "../../entity/datacontract/slaValidation.json", + "default": null + }, + "incidentId": { + "description": "Incident ID if the contract execution failed and an incident was created.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "executionTime": { + "description": "Time taken to execute the contract validation in milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + } + }, + "required": ["dataContractFQN", "timestamp", "contractExecutionStatus"], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/dataContract.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/dataContract.json index ca49effaf88..58dccd66af4 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/dataContract.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/dataContract.json @@ -31,10 +31,6 @@ } ] }, - "schemaField": { - "description": "Field defined in the data contract's schema.", - "$ref": "../../type/schema.json#/definitions/field" - }, "qualityExpectation": { "type": "object", "javaType": "org.openmetadata.schema.type.QualityExpectation", @@ -173,7 +169,7 @@ "description": "Schema definition for the data contract.", "type": "array", "items": { - "$ref": "#/definitions/schemaField" + "$ref": "./table.json#/definitions/column" }, "default": null }, @@ -206,6 +202,11 @@ "$ref": "../../type/entityReferenceList.json", "default": null }, + "reviewers": { + "description": "User references of the reviewers for this data contract.", + "$ref": "../../type/entityReferenceList.json", + "default": null + }, "effectiveFrom": { "description": "Date from which this data contract is effective.", "$ref": "../../type/basic.json#/definitions/dateTime", @@ -232,6 +233,74 @@ "sourceUrl": { "description": "Source URL of the data contract.", "$ref": "../../type/basic.json#/definitions/sourceUrl" + }, + "scheduleConfig": { + "description": "Configuration for scheduling data contract validation checks.", + "type": "object", + "properties": { + "enabled": { + "description": "Whether the scheduled validation is enabled.", + "type": "boolean", + "default": true + }, + "scheduleInterval": { + "description": "Schedule interval for validation checks in cron format (e.g., '0 0 * * *' for daily).", + "type": "string" + }, + "startDate": { + "description": "Start date for the scheduled validation.", + "$ref": "../../type/basic.json#/definitions/dateTime" + }, + "endDate": { + "description": "End date for the scheduled validation.", + "$ref": "../../type/basic.json#/definitions/dateTime" + }, + "timezone": { + "description": "Timezone for the scheduled validation.", + "type": "string", + "default": "UTC" + }, + "retries": { + "description": "Number of retries on validation failure.", + "type": "integer", + "default": 0, + "minimum": 0 + }, + "retryDelay": { + "description": "Delay between retries in seconds.", + "type": "integer", + "default": 300, + "minimum": 0 + }, + "timeout": { + "description": "Timeout for validation execution in seconds.", + "type": "integer", + "default": 3600, + "minimum": 1 + } + }, + "required": ["scheduleInterval"], + "additionalProperties": false + }, + "latestResult": { + "description": "Latest validation result for this data contract.", + "type": "object", + "properties": { + "timestamp": { + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "status": { + "type": "string", + "enum": ["Success", "Failed", "PartialSuccess", "Aborted", "Queued"] + }, + "message": { + "type": "string" + }, + "resultId": { + "$ref": "../../type/basic.json#/definitions/uuid" + } + }, + "additionalProperties": false } }, "required": [ diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/dataContractResult.json b/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/dataContractResult.json new file mode 100644 index 00000000000..425cc3bbe09 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/dataContractResult.json @@ -0,0 +1,61 @@ +{ + "$id": "https://open-metadata.org/schema/entity/datacontract/dataContractResult.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DataContractResult", + "description": "Schema to capture data contract execution results over time.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.datacontract.DataContractResult", + "javaInterfaces": ["org.openmetadata.schema.EntityTimeSeriesInterface"], + "properties": { + "id": { + "description": "Unique identifier of this data contract result instance.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "dataContractFQN": { + "description": "Fully qualified name of the data contract.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "timestamp": { + "description": "Timestamp when the data contract was executed.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "contractExecutionStatus": { + "description": "Overall status of the contract execution.", + "$ref": "../../type/contractExecutionStatus.json" + }, + "result": { + "description": "Detailed result of the data contract execution.", + "type": "string" + }, + "schemaValidation": { + "description": "Schema validation details.", + "$ref": "schemaValidation.json", + "default": null + }, + "semanticsValidation": { + "description": "Semantics validation details.", + "$ref": "semanticsValidation.json", + "default": null + }, + "qualityValidation": { + "description": "Quality expectations validation details.", + "$ref": "qualityValidation.json", + "default": null + }, + "slaValidation": { + "description": "SLA validation details.", + "$ref": "slaValidation.json", + "default": null + }, + "incidentId": { + "description": "Incident ID if the contract execution failed and an incident was created.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "executionTime": { + "description": "Time taken to execute the contract validation in milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + } + }, + "required": ["dataContractFQN", "timestamp", "contractExecutionStatus"], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/qualityValidation.json b/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/qualityValidation.json new file mode 100644 index 00000000000..ece32e53e66 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/qualityValidation.json @@ -0,0 +1,29 @@ +{ + "$id": "https://open-metadata.org/schema/entity/datacontract/qualityValidation.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "QualityValidation", + "description": "Quality validation details for data contract.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.datacontract.QualityValidation", + "properties": { + "passed": { + "description": "Number of quality checks passed.", + "type": "integer" + }, + "failed": { + "description": "Number of quality checks failed.", + "type": "integer" + }, + "total": { + "description": "Total number of quality checks.", + "type": "integer" + }, + "qualityScore": { + "description": "Overall quality score (0-100).", + "type": "number", + "minimum": 0, + "maximum": 100 + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/schemaValidation.json b/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/schemaValidation.json new file mode 100644 index 00000000000..1306a224417 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/schemaValidation.json @@ -0,0 +1,30 @@ +{ + "$id": "https://open-metadata.org/schema/entity/datacontract/schemaValidation.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SchemaValidation", + "description": "Schema validation details for data contract.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.datacontract.SchemaValidation", + "properties": { + "passed": { + "description": "Number of schema checks passed.", + "type": "integer" + }, + "failed": { + "description": "Number of schema checks failed.", + "type": "integer" + }, + "total": { + "description": "Total number of schema checks.", + "type": "integer" + }, + "failedFields": { + "description": "List of fields that failed validation.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/semanticsValidation.json b/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/semanticsValidation.json new file mode 100644 index 00000000000..5d0c588acab --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/semanticsValidation.json @@ -0,0 +1,39 @@ +{ + "$id": "https://open-metadata.org/schema/entity/datacontract/semanticsValidation.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SemanticsValidation", + "description": "Semantics validation details for data contract.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.datacontract.SemanticsValidation", + "properties": { + "passed": { + "description": "Number of semantics rules passed.", + "type": "integer" + }, + "failed": { + "description": "Number of semantics rules failed.", + "type": "integer" + }, + "total": { + "description": "Total number of semantics rules.", + "type": "integer" + }, + "failedRules": { + "description": "List of rules that failed validation.", + "type": "array", + "items": { + "type": "object", + "properties": { + "ruleName": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/slaValidation.json b/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/slaValidation.json new file mode 100644 index 00000000000..37c38cd4474 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/datacontract/slaValidation.json @@ -0,0 +1,27 @@ +{ + "$id": "https://open-metadata.org/schema/entity/datacontract/slaValidation.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SlaValidation", + "description": "SLA validation details for data contract.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.datacontract.SlaValidation", + "properties": { + "refreshFrequencyMet": { + "description": "Whether refresh frequency requirement was met.", + "type": "boolean" + }, + "latencyMet": { + "description": "Whether latency requirement was met.", + "type": "boolean" + }, + "availabilityMet": { + "description": "Whether availability requirement was met.", + "type": "boolean" + }, + "actualLatency": { + "description": "Actual latency in milliseconds.", + "type": "integer" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/type/contractExecutionStatus.json b/openmetadata-spec/src/main/resources/json/schema/type/contractExecutionStatus.json new file mode 100644 index 00000000000..96b6b498d10 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/type/contractExecutionStatus.json @@ -0,0 +1,32 @@ +{ + "$id": "https://open-metadata.org/schema/type/contractExecutionStatus.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ContractExecutionStatus", + "javaType": "org.openmetadata.schema.type.ContractExecutionStatus", + "description": "Status of the data contract execution.", + "type": "string", + "enum": [ + "Success", + "Failed", + "PartialSuccess", + "Aborted", + "Queued" + ], + "javaEnums": [ + { + "name": "Success" + }, + { + "name": "Failed" + }, + { + "name": "PartialSuccess" + }, + { + "name": "Aborted" + }, + { + "name": "Queued" + } + ] +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDataContract.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDataContract.ts index b1143e4b9d0..441f69617d6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDataContract.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDataContract.ts @@ -46,10 +46,18 @@ export interface CreateDataContract { * Quality expectations defined in the data contract. */ qualityExpectations?: QualityExpectation[]; + /** + * User references of the reviewers for this data contract. + */ + reviewers?: EntityReference[]; + /** + * Configuration for scheduling data contract validation checks. + */ + scheduleConfig?: ScheduleConfig; /** * Schema definition for the data contract. */ - schema?: Field[]; + schema?: Column[]; /** * Semantics rules defined in the data contract. */ @@ -144,20 +152,73 @@ export interface QualityExpectation { } /** - * Field defined in the data contract's schema. - * - * This schema defines the nested object to capture protobuf/avro/jsonschema of topic's - * schema. + * Configuration for scheduling data contract validation checks. */ -export interface Field { +export interface ScheduleConfig { /** - * Child fields if dataType or arrayDataType is `map`, `record`, `message` + * Whether the scheduled validation is enabled. */ - children?: Field[]; + enabled?: boolean; /** - * Data type of the field (int, date etc.). + * End date for the scheduled validation. */ - dataType: DataTypeTopic; + endDate?: Date; + /** + * Number of retries on validation failure. + */ + retries?: number; + /** + * Delay between retries in seconds. + */ + retryDelay?: number; + /** + * Schedule interval for validation checks in cron format (e.g., '0 0 * * *' for daily). + */ + scheduleInterval: string; + /** + * Start date for the scheduled validation. + */ + startDate?: Date; + /** + * Timeout for validation execution in seconds. + */ + timeout?: number; + /** + * Timezone for the scheduled validation. + */ + timezone?: string; +} + +/** + * This schema defines the type for a column in a table. + */ +export interface Column { + /** + * Data type used array in dataType. For example, `array` has dataType as `array` and + * arrayDataType as `int`. + */ + arrayDataType?: DataType; + /** + * Child columns if dataType or arrayDataType is `map`, `struct`, or `union` else `null`. + */ + children?: Column[]; + /** + * Column level constraint. + */ + constraint?: Constraint; + /** + * List of Custom Metrics registered for a table. + */ + customMetrics?: CustomMetric[]; + /** + * Length of `char`, `varchar`, `binary`, `varbinary` `dataTypes`, else null. For example, + * `varchar(20)` has dataType as `varchar` and dataLength as `20`. + */ + dataLength?: number; + /** + * Data type of the column (int, date etc.). + */ + dataType: DataType; /** * Display name used for dataType. This is useful for complex types, such as `array`, * `map`, `struct<>`, and union types. @@ -168,11 +229,36 @@ export interface Field { */ description?: string; /** - * Display Name that identifies this field name. + * Display Name that identifies this column name. */ displayName?: string; fullyQualifiedName?: string; - name: string; + /** + * Json schema only if the dataType is JSON else null. + */ + jsonSchema?: string; + name: string; + /** + * Ordinal position of the column. + */ + ordinalPosition?: number; + /** + * The precision of a numeric is the total count of significant digits in the whole number, + * that is, the number of digits to both sides of the decimal point. Precision is applicable + * Integer types, such as `INT`, `SMALLINT`, `BIGINT`, etc. It also applies to other Numeric + * types, such as `NUMBER`, `DECIMAL`, `DOUBLE`, `FLOAT`, etc. + */ + precision?: number; + /** + * Latest Data profile for a Column. + */ + profile?: ColumnProfile; + /** + * The scale of a numeric is the count of decimal digits in the fractional part, to the + * right of the decimal point. For Integer types, the scale is `0`. It mainly applies to non + * Integer Numeric types, such as `NUMBER`, `DECIMAL`, `DOUBLE`, `FLOAT`, etc. + */ + scale?: number; /** * Tags associated with the column. */ @@ -180,31 +266,299 @@ export interface Field { } /** - * Data type of the field (int, date etc.). + * Data type used array in dataType. For example, `array` has dataType as `array` and + * arrayDataType as `int`. * - * This enum defines the type of data defined in schema. + * This enum defines the type of data stored in a column. + * + * Data type of the column (int, date etc.). */ -export enum DataTypeTopic { +export enum DataType { + AggState = "AGG_STATE", + Aggregatefunction = "AGGREGATEFUNCTION", Array = "ARRAY", + Bigint = "BIGINT", + Binary = "BINARY", + Bit = "BIT", + Bitmap = "BITMAP", + Blob = "BLOB", Boolean = "BOOLEAN", + Bytea = "BYTEA", + Byteint = "BYTEINT", Bytes = "BYTES", + CIDR = "CIDR", + Char = "CHAR", + Clob = "CLOB", Date = "DATE", + Datetime = "DATETIME", + Datetimerange = "DATETIMERANGE", + Decimal = "DECIMAL", Double = "DOUBLE", Enum = "ENUM", Error = "ERROR", Fixed = "FIXED", Float = "FLOAT", + Geography = "GEOGRAPHY", + Geometry = "GEOMETRY", + Heirarchy = "HEIRARCHY", + Hll = "HLL", + Hllsketch = "HLLSKETCH", + Image = "IMAGE", + Inet = "INET", Int = "INT", + Interval = "INTERVAL", + Ipv4 = "IPV4", + Ipv6 = "IPV6", + JSON = "JSON", + Kpi = "KPI", + Largeint = "LARGEINT", Long = "LONG", + Longblob = "LONGBLOB", + Lowcardinality = "LOWCARDINALITY", + Macaddr = "MACADDR", Map = "MAP", + Measure = "MEASURE", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", + Mediumblob = "MEDIUMBLOB", + Mediumtext = "MEDIUMTEXT", + Money = "MONEY", + Ntext = "NTEXT", Null = "NULL", + Number = "NUMBER", + Numeric = "NUMERIC", + PGLsn = "PG_LSN", + PGSnapshot = "PG_SNAPSHOT", + Point = "POINT", + Polygon = "POLYGON", + QuantileState = "QUANTILE_STATE", Record = "RECORD", + Rowid = "ROWID", + Set = "SET", + Smallint = "SMALLINT", + Spatial = "SPATIAL", String = "STRING", + Struct = "STRUCT", + Super = "SUPER", + Table = "TABLE", + Text = "TEXT", Time = "TIME", Timestamp = "TIMESTAMP", Timestampz = "TIMESTAMPZ", + Tinyint = "TINYINT", + Tsquery = "TSQUERY", + Tsvector = "TSVECTOR", + Tuple = "TUPLE", + TxidSnapshot = "TXID_SNAPSHOT", + UUID = "UUID", + Uint = "UINT", Union = "UNION", Unknown = "UNKNOWN", + Varbinary = "VARBINARY", + Varchar = "VARCHAR", + Variant = "VARIANT", + XML = "XML", + Year = "YEAR", +} + +/** + * Column level constraint. + * + * This enum defines the type for column constraint. + */ +export enum Constraint { + NotNull = "NOT_NULL", + Null = "NULL", + PrimaryKey = "PRIMARY_KEY", + Unique = "UNIQUE", +} + +/** + * Custom Metric definition that we will associate with a column. + */ +export interface CustomMetric { + /** + * Name of the column in a table. + */ + columnName?: string; + /** + * Description of the Metric. + */ + description?: string; + /** + * SQL expression to compute the Metric. It should return a single numerical value. + */ + expression: string; + /** + * Unique identifier of this Custom Metric instance. + */ + id?: string; + /** + * Name that identifies this Custom Metric. + */ + name: string; + /** + * Owners of this Custom Metric. + */ + owners?: EntityReference[]; + /** + * Last update time corresponding to the new version of the entity in Unix epoch time + * milliseconds. + */ + updatedAt?: number; + /** + * User who made the update. + */ + updatedBy?: string; +} + +/** + * Latest Data profile for a Column. + * + * This schema defines the type to capture the table's column profile. + */ +export interface ColumnProfile { + /** + * Custom Metrics profile list bound to a column. + */ + customMetrics?: CustomMetricProfile[]; + /** + * Number of values that contain distinct values. + */ + distinctCount?: number; + /** + * Proportion of distinct values in a column. + */ + distinctProportion?: number; + /** + * No.of Rows that contain duplicates in a column. + */ + duplicateCount?: number; + /** + * First quartile of a column. + */ + firstQuartile?: number; + /** + * Histogram of a column. + */ + histogram?: any[] | boolean | HistogramClass | number | number | null | string; + /** + * Inter quartile range of a column. + */ + interQuartileRange?: number; + /** + * Maximum value in a column. + */ + max?: number | string; + /** + * Maximum string length in a column. + */ + maxLength?: number; + /** + * Avg value in a column. + */ + mean?: number; + /** + * Median of a column. + */ + median?: number; + /** + * Minimum value in a column. + */ + min?: number | string; + /** + * Minimum string length in a column. + */ + minLength?: number; + /** + * Missing count is calculated by subtracting valuesCount - validCount. + */ + missingCount?: number; + /** + * Missing Percentage is calculated by taking percentage of validCount/valuesCount. + */ + missingPercentage?: number; + /** + * Column Name. + */ + name: string; + /** + * Non parametric skew of a column. + */ + nonParametricSkew?: number; + /** + * No.of null values in a column. + */ + nullCount?: number; + /** + * No.of null value proportion in columns. + */ + nullProportion?: number; + /** + * Standard deviation of a column. + */ + stddev?: number; + /** + * Median value in a column. + */ + sum?: number; + /** + * First quartile of a column. + */ + thirdQuartile?: number; + /** + * Timestamp on which profile is taken. + */ + timestamp: number; + /** + * No. of unique values in the column. + */ + uniqueCount?: number; + /** + * Proportion of number of unique values in a column. + */ + uniqueProportion?: number; + /** + * Total count of valid values in this column. + */ + validCount?: number; + /** + * Total count of the values in this column. + */ + valuesCount?: number; + /** + * Percentage of values in this column with respect to row count. + */ + valuesPercentage?: number; + /** + * Variance of a column. + */ + variance?: number; +} + +/** + * Profiling results of a Custom Metric. + */ +export interface CustomMetricProfile { + /** + * Custom metric name. + */ + name?: string; + /** + * Profiling results for the metric. + */ + value?: number; +} + +export interface HistogramClass { + /** + * Boundaries of Histogram. + */ + boundaries?: any[]; + /** + * Frequencies of Histogram. + */ + frequencies?: any[]; } /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDataContractResult.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDataContractResult.ts new file mode 100644 index 00000000000..5e92cb8afb0 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDataContractResult.ts @@ -0,0 +1,171 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Create data contract result API request. + */ +export interface CreateDataContractResult { + /** + * Overall status of the contract execution. + */ + contractExecutionStatus: ContractExecutionStatus; + /** + * Fully qualified name of the data contract. + */ + dataContractFQN: string; + /** + * Time taken to execute the contract validation in milliseconds. + */ + executionTime?: number; + /** + * Incident ID if the contract execution failed and an incident was created. + */ + incidentId?: string; + /** + * Quality expectations validation details. + */ + qualityValidation?: QualityValidation; + /** + * Detailed result of the data contract execution. + */ + result?: string; + /** + * Schema validation details. + */ + schemaValidation?: SchemaValidation; + /** + * Semantics validation details. + */ + semanticsValidation?: SemanticsValidation; + /** + * SLA validation details. + */ + slaValidation?: SlaValidation; + /** + * Timestamp when the data contract was executed. + */ + timestamp: number; +} + +/** + * Overall status of the contract execution. + * + * Status of the data contract execution. + */ +export enum ContractExecutionStatus { + Aborted = "Aborted", + Failed = "Failed", + PartialSuccess = "PartialSuccess", + Queued = "Queued", + Success = "Success", +} + +/** + * Quality expectations validation details. + * + * Quality validation details for data contract. + */ +export interface QualityValidation { + /** + * Number of quality checks failed. + */ + failed?: number; + /** + * Number of quality checks passed. + */ + passed?: number; + /** + * Overall quality score (0-100). + */ + qualityScore?: number; + /** + * Total number of quality checks. + */ + total?: number; +} + +/** + * Schema validation details. + * + * Schema validation details for data contract. + */ +export interface SchemaValidation { + /** + * Number of schema checks failed. + */ + failed?: number; + /** + * List of fields that failed validation. + */ + failedFields?: string[]; + /** + * Number of schema checks passed. + */ + passed?: number; + /** + * Total number of schema checks. + */ + total?: number; +} + +/** + * Semantics validation details. + * + * Semantics validation details for data contract. + */ +export interface SemanticsValidation { + /** + * Number of semantics rules failed. + */ + failed?: number; + /** + * List of rules that failed validation. + */ + failedRules?: FailedRule[]; + /** + * Number of semantics rules passed. + */ + passed?: number; + /** + * Total number of semantics rules. + */ + total?: number; +} + +export interface FailedRule { + reason?: string; + ruleName?: string; +} + +/** + * SLA validation details. + * + * SLA validation details for data contract. + */ +export interface SlaValidation { + /** + * Actual latency in milliseconds. + */ + actualLatency?: number; + /** + * Whether availability requirement was met. + */ + availabilityMet?: boolean; + /** + * Whether latency requirement was met. + */ + latencyMet?: boolean; + /** + * Whether refresh frequency requirement was met. + */ + refreshFrequencyMet?: boolean; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dataContract.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dataContract.ts index ea5be5d91d4..f07c2e21cba 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dataContract.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dataContract.ts @@ -62,6 +62,10 @@ export interface DataContract { * Incremental change description of the entity. */ incrementalChangeDescription?: ChangeDescription; + /** + * Latest validation result for this data contract. + */ + latestResult?: LatestResult; /** * Name of the data contract. */ @@ -74,10 +78,18 @@ export interface DataContract { * Quality expectations defined in the data contract. */ qualityExpectations?: QualityExpectation[]; + /** + * User references of the reviewers for this data contract. + */ + reviewers?: EntityReference[]; + /** + * Configuration for scheduling data contract validation checks. + */ + scheduleConfig?: ScheduleConfig; /** * Schema definition for the data contract. */ - schema?: Field[]; + schema?: Column[]; /** * Semantics rules defined in the data contract. */ @@ -251,6 +263,24 @@ export interface EntityReference { type: string; } +/** + * Latest validation result for this data contract. + */ +export interface LatestResult { + message?: string; + resultId?: string; + status?: Status; + timestamp?: number; +} + +export enum Status { + Aborted = "Aborted", + Failed = "Failed", + PartialSuccess = "PartialSuccess", + Queued = "Queued", + Success = "Success", +} + /** * Quality expectation defined in the data contract. */ @@ -274,20 +304,73 @@ export interface QualityExpectation { } /** - * Field defined in the data contract's schema. - * - * This schema defines the nested object to capture protobuf/avro/jsonschema of topic's - * schema. + * Configuration for scheduling data contract validation checks. */ -export interface Field { +export interface ScheduleConfig { /** - * Child fields if dataType or arrayDataType is `map`, `record`, `message` + * Whether the scheduled validation is enabled. */ - children?: Field[]; + enabled?: boolean; /** - * Data type of the field (int, date etc.). + * End date for the scheduled validation. */ - dataType: DataTypeTopic; + endDate?: Date; + /** + * Number of retries on validation failure. + */ + retries?: number; + /** + * Delay between retries in seconds. + */ + retryDelay?: number; + /** + * Schedule interval for validation checks in cron format (e.g., '0 0 * * *' for daily). + */ + scheduleInterval: string; + /** + * Start date for the scheduled validation. + */ + startDate?: Date; + /** + * Timeout for validation execution in seconds. + */ + timeout?: number; + /** + * Timezone for the scheduled validation. + */ + timezone?: string; +} + +/** + * This schema defines the type for a column in a table. + */ +export interface Column { + /** + * Data type used array in dataType. For example, `array` has dataType as `array` and + * arrayDataType as `int`. + */ + arrayDataType?: DataType; + /** + * Child columns if dataType or arrayDataType is `map`, `struct`, or `union` else `null`. + */ + children?: Column[]; + /** + * Column level constraint. + */ + constraint?: Constraint; + /** + * List of Custom Metrics registered for a table. + */ + customMetrics?: CustomMetric[]; + /** + * Length of `char`, `varchar`, `binary`, `varbinary` `dataTypes`, else null. For example, + * `varchar(20)` has dataType as `varchar` and dataLength as `20`. + */ + dataLength?: number; + /** + * Data type of the column (int, date etc.). + */ + dataType: DataType; /** * Display name used for dataType. This is useful for complex types, such as `array`, * `map`, `struct<>`, and union types. @@ -298,11 +381,36 @@ export interface Field { */ description?: string; /** - * Display Name that identifies this field name. + * Display Name that identifies this column name. */ displayName?: string; fullyQualifiedName?: string; - name: string; + /** + * Json schema only if the dataType is JSON else null. + */ + jsonSchema?: string; + name: string; + /** + * Ordinal position of the column. + */ + ordinalPosition?: number; + /** + * The precision of a numeric is the total count of significant digits in the whole number, + * that is, the number of digits to both sides of the decimal point. Precision is applicable + * Integer types, such as `INT`, `SMALLINT`, `BIGINT`, etc. It also applies to other Numeric + * types, such as `NUMBER`, `DECIMAL`, `DOUBLE`, `FLOAT`, etc. + */ + precision?: number; + /** + * Latest Data profile for a Column. + */ + profile?: ColumnProfile; + /** + * The scale of a numeric is the count of decimal digits in the fractional part, to the + * right of the decimal point. For Integer types, the scale is `0`. It mainly applies to non + * Integer Numeric types, such as `NUMBER`, `DECIMAL`, `DOUBLE`, `FLOAT`, etc. + */ + scale?: number; /** * Tags associated with the column. */ @@ -310,31 +418,299 @@ export interface Field { } /** - * Data type of the field (int, date etc.). + * Data type used array in dataType. For example, `array` has dataType as `array` and + * arrayDataType as `int`. * - * This enum defines the type of data defined in schema. + * This enum defines the type of data stored in a column. + * + * Data type of the column (int, date etc.). */ -export enum DataTypeTopic { +export enum DataType { + AggState = "AGG_STATE", + Aggregatefunction = "AGGREGATEFUNCTION", Array = "ARRAY", + Bigint = "BIGINT", + Binary = "BINARY", + Bit = "BIT", + Bitmap = "BITMAP", + Blob = "BLOB", Boolean = "BOOLEAN", + Bytea = "BYTEA", + Byteint = "BYTEINT", Bytes = "BYTES", + CIDR = "CIDR", + Char = "CHAR", + Clob = "CLOB", Date = "DATE", + Datetime = "DATETIME", + Datetimerange = "DATETIMERANGE", + Decimal = "DECIMAL", Double = "DOUBLE", Enum = "ENUM", Error = "ERROR", Fixed = "FIXED", Float = "FLOAT", + Geography = "GEOGRAPHY", + Geometry = "GEOMETRY", + Heirarchy = "HEIRARCHY", + Hll = "HLL", + Hllsketch = "HLLSKETCH", + Image = "IMAGE", + Inet = "INET", Int = "INT", + Interval = "INTERVAL", + Ipv4 = "IPV4", + Ipv6 = "IPV6", + JSON = "JSON", + Kpi = "KPI", + Largeint = "LARGEINT", Long = "LONG", + Longblob = "LONGBLOB", + Lowcardinality = "LOWCARDINALITY", + Macaddr = "MACADDR", Map = "MAP", + Measure = "MEASURE", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", + Mediumblob = "MEDIUMBLOB", + Mediumtext = "MEDIUMTEXT", + Money = "MONEY", + Ntext = "NTEXT", Null = "NULL", + Number = "NUMBER", + Numeric = "NUMERIC", + PGLsn = "PG_LSN", + PGSnapshot = "PG_SNAPSHOT", + Point = "POINT", + Polygon = "POLYGON", + QuantileState = "QUANTILE_STATE", Record = "RECORD", + Rowid = "ROWID", + Set = "SET", + Smallint = "SMALLINT", + Spatial = "SPATIAL", String = "STRING", + Struct = "STRUCT", + Super = "SUPER", + Table = "TABLE", + Text = "TEXT", Time = "TIME", Timestamp = "TIMESTAMP", Timestampz = "TIMESTAMPZ", + Tinyint = "TINYINT", + Tsquery = "TSQUERY", + Tsvector = "TSVECTOR", + Tuple = "TUPLE", + TxidSnapshot = "TXID_SNAPSHOT", + UUID = "UUID", + Uint = "UINT", Union = "UNION", Unknown = "UNKNOWN", + Varbinary = "VARBINARY", + Varchar = "VARCHAR", + Variant = "VARIANT", + XML = "XML", + Year = "YEAR", +} + +/** + * Column level constraint. + * + * This enum defines the type for column constraint. + */ +export enum Constraint { + NotNull = "NOT_NULL", + Null = "NULL", + PrimaryKey = "PRIMARY_KEY", + Unique = "UNIQUE", +} + +/** + * Custom Metric definition that we will associate with a column. + */ +export interface CustomMetric { + /** + * Name of the column in a table. + */ + columnName?: string; + /** + * Description of the Metric. + */ + description?: string; + /** + * SQL expression to compute the Metric. It should return a single numerical value. + */ + expression: string; + /** + * Unique identifier of this Custom Metric instance. + */ + id?: string; + /** + * Name that identifies this Custom Metric. + */ + name: string; + /** + * Owners of this Custom Metric. + */ + owners?: EntityReference[]; + /** + * Last update time corresponding to the new version of the entity in Unix epoch time + * milliseconds. + */ + updatedAt?: number; + /** + * User who made the update. + */ + updatedBy?: string; +} + +/** + * Latest Data profile for a Column. + * + * This schema defines the type to capture the table's column profile. + */ +export interface ColumnProfile { + /** + * Custom Metrics profile list bound to a column. + */ + customMetrics?: CustomMetricProfile[]; + /** + * Number of values that contain distinct values. + */ + distinctCount?: number; + /** + * Proportion of distinct values in a column. + */ + distinctProportion?: number; + /** + * No.of Rows that contain duplicates in a column. + */ + duplicateCount?: number; + /** + * First quartile of a column. + */ + firstQuartile?: number; + /** + * Histogram of a column. + */ + histogram?: any[] | boolean | HistogramClass | number | number | null | string; + /** + * Inter quartile range of a column. + */ + interQuartileRange?: number; + /** + * Maximum value in a column. + */ + max?: number | string; + /** + * Maximum string length in a column. + */ + maxLength?: number; + /** + * Avg value in a column. + */ + mean?: number; + /** + * Median of a column. + */ + median?: number; + /** + * Minimum value in a column. + */ + min?: number | string; + /** + * Minimum string length in a column. + */ + minLength?: number; + /** + * Missing count is calculated by subtracting valuesCount - validCount. + */ + missingCount?: number; + /** + * Missing Percentage is calculated by taking percentage of validCount/valuesCount. + */ + missingPercentage?: number; + /** + * Column Name. + */ + name: string; + /** + * Non parametric skew of a column. + */ + nonParametricSkew?: number; + /** + * No.of null values in a column. + */ + nullCount?: number; + /** + * No.of null value proportion in columns. + */ + nullProportion?: number; + /** + * Standard deviation of a column. + */ + stddev?: number; + /** + * Median value in a column. + */ + sum?: number; + /** + * First quartile of a column. + */ + thirdQuartile?: number; + /** + * Timestamp on which profile is taken. + */ + timestamp: number; + /** + * No. of unique values in the column. + */ + uniqueCount?: number; + /** + * Proportion of number of unique values in a column. + */ + uniqueProportion?: number; + /** + * Total count of valid values in this column. + */ + validCount?: number; + /** + * Total count of the values in this column. + */ + valuesCount?: number; + /** + * Percentage of values in this column with respect to row count. + */ + valuesPercentage?: number; + /** + * Variance of a column. + */ + variance?: number; +} + +/** + * Profiling results of a Custom Metric. + */ +export interface CustomMetricProfile { + /** + * Custom metric name. + */ + name?: string; + /** + * Profiling results for the metric. + */ + value?: number; +} + +export interface HistogramClass { + /** + * Boundaries of Histogram. + */ + boundaries?: any[]; + /** + * Frequencies of Histogram. + */ + frequencies?: any[]; } /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/dataContractResult.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/dataContractResult.ts new file mode 100644 index 00000000000..2e72c965cfe --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/dataContractResult.ts @@ -0,0 +1,175 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Schema to capture data contract execution results over time. + */ +export interface DataContractResult { + /** + * Overall status of the contract execution. + */ + contractExecutionStatus: ContractExecutionStatus; + /** + * Fully qualified name of the data contract. + */ + dataContractFQN: string; + /** + * Time taken to execute the contract validation in milliseconds. + */ + executionTime?: number; + /** + * Unique identifier of this data contract result instance. + */ + id?: string; + /** + * Incident ID if the contract execution failed and an incident was created. + */ + incidentId?: string; + /** + * Quality expectations validation details. + */ + qualityValidation?: QualityValidation; + /** + * Detailed result of the data contract execution. + */ + result?: string; + /** + * Schema validation details. + */ + schemaValidation?: SchemaValidation; + /** + * Semantics validation details. + */ + semanticsValidation?: SemanticsValidation; + /** + * SLA validation details. + */ + slaValidation?: SlaValidation; + /** + * Timestamp when the data contract was executed. + */ + timestamp: number; +} + +/** + * Overall status of the contract execution. + * + * Status of the data contract execution. + */ +export enum ContractExecutionStatus { + Aborted = "Aborted", + Failed = "Failed", + PartialSuccess = "PartialSuccess", + Queued = "Queued", + Success = "Success", +} + +/** + * Quality expectations validation details. + * + * Quality validation details for data contract. + */ +export interface QualityValidation { + /** + * Number of quality checks failed. + */ + failed?: number; + /** + * Number of quality checks passed. + */ + passed?: number; + /** + * Overall quality score (0-100). + */ + qualityScore?: number; + /** + * Total number of quality checks. + */ + total?: number; +} + +/** + * Schema validation details. + * + * Schema validation details for data contract. + */ +export interface SchemaValidation { + /** + * Number of schema checks failed. + */ + failed?: number; + /** + * List of fields that failed validation. + */ + failedFields?: string[]; + /** + * Number of schema checks passed. + */ + passed?: number; + /** + * Total number of schema checks. + */ + total?: number; +} + +/** + * Semantics validation details. + * + * Semantics validation details for data contract. + */ +export interface SemanticsValidation { + /** + * Number of semantics rules failed. + */ + failed?: number; + /** + * List of rules that failed validation. + */ + failedRules?: FailedRule[]; + /** + * Number of semantics rules passed. + */ + passed?: number; + /** + * Total number of semantics rules. + */ + total?: number; +} + +export interface FailedRule { + reason?: string; + ruleName?: string; +} + +/** + * SLA validation details. + * + * SLA validation details for data contract. + */ +export interface SlaValidation { + /** + * Actual latency in milliseconds. + */ + actualLatency?: number; + /** + * Whether availability requirement was met. + */ + availabilityMet?: boolean; + /** + * Whether latency requirement was met. + */ + latencyMet?: boolean; + /** + * Whether refresh frequency requirement was met. + */ + refreshFrequencyMet?: boolean; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/qualityValidation.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/qualityValidation.ts new file mode 100644 index 00000000000..6b6ebd17832 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/qualityValidation.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Quality validation details for data contract. + */ +export interface QualityValidation { + /** + * Number of quality checks failed. + */ + failed?: number; + /** + * Number of quality checks passed. + */ + passed?: number; + /** + * Overall quality score (0-100). + */ + qualityScore?: number; + /** + * Total number of quality checks. + */ + total?: number; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/schemaValidation.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/schemaValidation.ts new file mode 100644 index 00000000000..2f94c8fa696 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/schemaValidation.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Schema validation details for data contract. + */ +export interface SchemaValidation { + /** + * Number of schema checks failed. + */ + failed?: number; + /** + * List of fields that failed validation. + */ + failedFields?: string[]; + /** + * Number of schema checks passed. + */ + passed?: number; + /** + * Total number of schema checks. + */ + total?: number; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/semanticsValidation.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/semanticsValidation.ts new file mode 100644 index 00000000000..78f6756fcc9 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/semanticsValidation.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Semantics validation details for data contract. + */ +export interface SemanticsValidation { + /** + * Number of semantics rules failed. + */ + failed?: number; + /** + * List of rules that failed validation. + */ + failedRules?: FailedRule[]; + /** + * Number of semantics rules passed. + */ + passed?: number; + /** + * Total number of semantics rules. + */ + total?: number; +} + +export interface FailedRule { + reason?: string; + ruleName?: string; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/slaValidation.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/slaValidation.ts new file mode 100644 index 00000000000..9d96bbbafa8 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/datacontract/slaValidation.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * SLA validation details for data contract. + */ +export interface SlaValidation { + /** + * Actual latency in milliseconds. + */ + actualLatency?: number; + /** + * Whether availability requirement was met. + */ + availabilityMet?: boolean; + /** + * Whether latency requirement was met. + */ + latencyMet?: boolean; + /** + * Whether refresh frequency requirement was met. + */ + refreshFrequencyMet?: boolean; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/type/contractExecutionStatus.ts b/openmetadata-ui/src/main/resources/ui/src/generated/type/contractExecutionStatus.ts new file mode 100644 index 00000000000..9bed6f8d37b --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/type/contractExecutionStatus.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Status of the data contract execution. + */ +export enum ContractExecutionStatus { + Aborted = "Aborted", + Failed = "Failed", + PartialSuccess = "PartialSuccess", + Queued = "Queued", + Success = "Success", +}