diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java index 48e7575f8aa..eec20b6f2c7 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java @@ -50,6 +50,7 @@ import org.openmetadata.catalog.entity.data.Database; import org.openmetadata.catalog.entity.data.Table; import org.openmetadata.catalog.entity.services.DatabaseService; import org.openmetadata.catalog.exception.CatalogExceptionMessage; +import org.openmetadata.catalog.exception.EntityNotFoundException; import org.openmetadata.catalog.resources.databases.TableResource; import org.openmetadata.catalog.tests.ColumnTest; import org.openmetadata.catalog.tests.TableTest; @@ -260,27 +261,23 @@ public class TableRepository extends EntityRepository { List storedTableTests = getTableTests(table); // we will override any test case name passed by user/client with tableName + testType // our assumption is there is only one instance of a test type as of now. - tableTest.setName(table.getName() + "." + tableTest.getTableTestCase().getTestType().toString()); + tableTest.setName(table.getName() + "." + tableTest.getTestCase().getTableTestType().toString()); Map storedMapTableTests = new HashMap<>(); if (storedTableTests != null) { for (TableTest t : storedTableTests) { storedMapTableTests.put(t.getName(), t); } } - // new test add UUID - if (!storedMapTableTests.containsKey(tableTest.getName())) { - tableTest.setId(UUID.randomUUID()); - } - - // process test result - if (storedMapTableTests.containsKey(tableTest.getName()) - && tableTest.getResults() != null - && !tableTest.getResults().isEmpty()) { + // existing test, use the previous UUID + if (storedMapTableTests.containsKey(tableTest.getName())) { TableTest prevTableTest = storedMapTableTests.get(tableTest.getName()); - List prevTestCaseResults = prevTableTest.getResults(); - List newTestCaseResults = tableTest.getResults(); - newTestCaseResults.addAll(prevTestCaseResults); - tableTest.setResults(newTestCaseResults); + tableTest.setId(prevTableTest.getId()); + // process test result + if (tableTest.getResults() != null && !tableTest.getResults().isEmpty()) { + List prevTestCaseResults = prevTableTest.getResults(); + prevTestCaseResults.addAll(tableTest.getResults()); + tableTest.setResults(prevTestCaseResults); + } } storedMapTableTests.put(tableTest.getName(), tableTest); @@ -289,7 +286,35 @@ public class TableRepository extends EntityRepository
{ .entityExtensionDAO() .insert(tableId.toString(), "table.tableTests", "tableTest", JsonUtils.pojoToJson(updatedTests)); setFields(table, Fields.EMPTY_FIELDS); - return table.withTableTests(getTableTests(table)); + // return the only test instead of querying all tests and results + return table.withTableTests(List.of(tableTest)); + } + + @Transaction + public Table deleteTableTest(UUID tableId, String tableTestType) throws IOException, ParseException { + // Validate the request content + Table table = daoCollection.tableDAO().findEntityById(tableId); + // if ID is not passed we treat it as a new test case being added + List storedTableTests = getTableTests(table); + // we will override any test case name passed by user/client with tableName + testType + // our assumption is there is only one instance of a test type as of now. + String tableTestName = table.getName() + "." + tableTestType; + Map storedMapTableTests = new HashMap<>(); + if (storedTableTests != null) { + for (TableTest t : storedTableTests) { + storedMapTableTests.put(t.getName(), t); + } + } + if (!storedMapTableTests.containsKey(tableTestName)) { + throw new EntityNotFoundException(String.format("Failed to find %s for %s", tableTestName, table.getName())); + } + TableTest deleteTableTest = storedMapTableTests.get(tableTestName); + storedMapTableTests.remove(tableTestName); + List updatedTests = new ArrayList<>(storedMapTableTests.values()); + daoCollection + .entityExtensionDAO() + .insert(tableId.toString(), "table.tableTests", "tableTest", JsonUtils.pojoToJson(updatedTests)); + return table.withTableTests(List.of(deleteTableTest)); } @Transaction @@ -300,7 +325,7 @@ public class TableRepository extends EntityRepository
{ validateColumn(table, columnName); // we will override any test case name passed by user/client with columnName + testType // our assumption is there is only one instance of a test type as of now. - columnTest.setName(columnName + "." + columnTest.getTestCase().getTestType().toString()); + columnTest.setName(columnName + "." + columnTest.getTestCase().getColumnTestType().toString()); List storedColumnTests = getColumnTests(table, columnName); Map storedMapColumnTests = new HashMap<>(); if (storedColumnTests != null) { @@ -309,20 +334,17 @@ public class TableRepository extends EntityRepository
{ } } - // new test, generate UUID - if (!storedMapColumnTests.containsKey(columnTest.getName())) { - columnTest.setId(UUID.randomUUID()); - } - - // process test result - if (storedMapColumnTests.containsKey(columnTest.getName()) - && columnTest.getResults() != null - && !columnTest.getResults().isEmpty()) { + // existingTest use the previous UUID + if (storedMapColumnTests.containsKey(columnTest.getName())) { ColumnTest prevColumnTest = storedMapColumnTests.get(columnTest.getName()); - List prevTestCaseResults = prevColumnTest.getResults(); - List newTestCaseResults = columnTest.getResults(); - newTestCaseResults.addAll(prevTestCaseResults); - columnTest.setResults(newTestCaseResults); + columnTest.setId(prevColumnTest.getId()); + + // process test results + if (columnTest.getResults() != null && !columnTest.getResults().isEmpty()) { + List prevTestCaseResults = prevColumnTest.getResults(); + prevTestCaseResults.addAll(columnTest.getResults()); + columnTest.setResults(prevTestCaseResults); + } } storedMapColumnTests.put(columnTest.getName(), columnTest); @@ -332,7 +354,48 @@ public class TableRepository extends EntityRepository
{ .entityExtensionDAO() .insert(table.getId().toString(), extension, "columnTest", JsonUtils.pojoToJson(updatedTests)); setFields(table, Fields.EMPTY_FIELDS); - getColumnTests(true, table); + // return the newly created/updated column test only + for (Column column : table.getColumns()) { + if (column.getName().equals(columnName)) { + column.setColumnTests(List.of(columnTest)); + } + } + return table; + } + + @Transaction + public Table deleteColumnTest(UUID tableId, String columnName, String columnTestType) throws IOException { + // Validate the request content + Table table = daoCollection.tableDAO().findEntityById(tableId); + validateColumn(table, columnName); + // we will override any test case name passed by user/client with columnName + testType + // our assumption is there is only one instance of a test type as of now. + String columnTestName = columnName + "." + columnTestType; + List storedColumnTests = getColumnTests(table, columnName); + Map storedMapColumnTests = new HashMap<>(); + if (storedColumnTests != null) { + for (ColumnTest ct : storedColumnTests) { + storedMapColumnTests.put(ct.getName(), ct); + } + } + + if (!storedMapColumnTests.containsKey(columnTestName)) { + throw new EntityNotFoundException(String.format("Failed to find %s for %s", columnTestName, table.getName())); + } + + ColumnTest deleteColumnTest = storedMapColumnTests.get(columnTestName); + storedMapColumnTests.remove(columnTestName); + List updatedTests = new ArrayList<>(storedMapColumnTests.values()); + String extension = "table.column." + columnName + ".tests"; + daoCollection + .entityExtensionDAO() + .insert(table.getId().toString(), extension, "columnTest", JsonUtils.pojoToJson(updatedTests)); + // return the newly created/updated column test only + for (Column column : table.getColumns()) { + if (column.getName().equals(columnName)) { + column.setColumnTests(List.of(deleteColumnTest)); + } + } return table; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java index 1a2310b7921..3d3a1d3cc9f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java @@ -52,6 +52,8 @@ import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.api.data.CreateTable; +import org.openmetadata.catalog.api.tests.CreateColumnTest; +import org.openmetadata.catalog.api.tests.CreateTableTest; import org.openmetadata.catalog.entity.data.Table; import org.openmetadata.catalog.jdbi3.CollectionDAO; import org.openmetadata.catalog.jdbi3.TableRepository; @@ -521,13 +523,29 @@ public class TableResource { @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id, - TableTest tableTest) + CreateTableTest createTableTest) throws IOException, ParseException { SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + TableTest tableTest = getTableTest(securityContext, createTableTest); Table table = dao.addTableTest(UUID.fromString(id), tableTest); return addHref(uriInfo, table); } + @DELETE + @Path("/{id}/tableTest/{tableTestType}") + @Operation(summary = "delete table test case", tags = "tables", description = "Delete test case from the table.") + public Table deleteTableTest( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id, + @Parameter(description = "Table Test Type", schema = @Schema(type = "string")) @PathParam("tableTestType") + String tableTestType) + throws IOException, ParseException { + SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + Table table = dao.deleteTableTest(UUID.fromString(id), tableTestType); + return addHref(uriInfo, table); + } + @PUT @Path("/{id}/columnTest") @Operation(summary = "Add table test cases", tags = "tables", description = "Add test cases to the table.") @@ -535,13 +553,34 @@ public class TableResource { @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id, - ColumnTest columnTest) + CreateColumnTest createColumnTest) throws IOException, ParseException { SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + ColumnTest columnTest = getColumnTest(securityContext, createColumnTest); Table table = dao.addColumnTest(UUID.fromString(id), columnTest); return addHref(uriInfo, table); } + @DELETE + @Path("/{id}/columnTest/{columnName}/{columnTestType}") + @Operation( + summary = "delete column test case", + tags = "tables", + description = "Delete column test case from the table.") + public Table deleteColumnTest( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id, + @Parameter(description = "column of the table", schema = @Schema(type = "string")) @PathParam("columnName") + String columnName, + @Parameter(description = "column Test Type", schema = @Schema(type = "string")) @PathParam("columnTestType") + String columnTestType) + throws IOException, ParseException { + SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + Table table = dao.deleteColumnTest(UUID.fromString(id), columnName, columnTestType); + return addHref(uriInfo, table); + } + @DELETE @Path("/{id}/followers/{userId}") @Operation( @@ -598,4 +637,29 @@ public class TableResource { .withUpdatedAt(System.currentTimeMillis()) .withDatabase(create.getDatabase()); } + + private TableTest getTableTest(SecurityContext securityContext, CreateTableTest create) { + return new TableTest() + .withId(UUID.randomUUID()) + .withDescription(create.getDescription()) + .withTestCase(create.getTestCase()) + .withOwner(create.getOwner()) + .withExecutionFrequency(create.getExecutionFrequency()) + .withResults(create.getResult() != null ? List.of(create.getResult()) : new ArrayList<>()) + .withUpdatedBy(securityContext.getUserPrincipal().getName()) + .withUpdatedAt(System.currentTimeMillis()); + } + + private ColumnTest getColumnTest(SecurityContext securityContext, CreateColumnTest create) { + return new ColumnTest() + .withId(UUID.randomUUID()) + .withDescription(create.getDescription()) + .withTestCase(create.getTestCase()) + .withColumnName(create.getColumnName()) + .withOwner(create.getOwner()) + .withExecutionFrequency(create.getExecutionFrequency()) + .withResults(create.getResult() != null ? List.of(create.getResult()) : new ArrayList<>()) + .withUpdatedBy(securityContext.getUserPrincipal().getName()) + .withUpdatedAt(System.currentTimeMillis()); + } } diff --git a/catalog-rest-service/src/main/resources/json/schema/api/tests/createColumnTest.json b/catalog-rest-service/src/main/resources/json/schema/api/tests/createColumnTest.json new file mode 100644 index 00000000000..057078c5608 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/api/tests/createColumnTest.json @@ -0,0 +1,41 @@ +{ + "$id": "https://open-metadata.org/schema/api/tests/columnTest.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateColumnTestRequest", + "description": "ColumnTest is a test definition to capture data quality tests against tables and columns.", + "type": "object", + "properties": { + "description": { + "description": "Description of the testcase.", + "type": "string" + }, + "columnName": { + "description": "Name of the column in a table.", + "type": "string" + }, + "testCase": { + "$ref": "../../tests/columnTest.json#/definitions/columnTestCase" + }, + "executionFrequency": { + "$ref": "../../tests/basic.json#/definitions/testCaseExecutionFrequency" + }, + "result": { + "$ref": "../../tests/basic.json#/definitions/testCaseResult" + }, + "owner": { + "description": "Owner of this Pipeline.", + "$ref": "../../type/entityReference.json", + "default": null + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + } + }, + "required": ["columnName", "testCase"], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/api/tests/createTableTest.json b/catalog-rest-service/src/main/resources/json/schema/api/tests/createTableTest.json new file mode 100644 index 00000000000..d636a5a97db --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/api/tests/createTableTest.json @@ -0,0 +1,37 @@ +{ + "$id": "https://open-metadata.org/schema/api/tests/tableTest.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateTableTestRequest", + "description": "TableTest is a test definition to capture data quality tests against tables and columns.", + "type": "object", + "properties": { + "description": { + "description": "Description of the testcase.", + "type": "string" + }, + "testCase": { + "$ref": "../../tests/tableTest.json#/definitions/tableTestCase" + }, + "executionFrequency": { + "$ref": "../../tests/basic.json#/definitions/testCaseExecutionFrequency" + }, + "result": { + "$ref": "../../tests/basic.json#/definitions/testCaseResult" + }, + "owner": { + "description": "Owner of this Pipeline.", + "$ref": "../../type/entityReference.json", + "default": null + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + } + }, + "required": ["testCase"], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/basic.json b/catalog-rest-service/src/main/resources/json/schema/tests/basic.json index ce458440beb..00ae71f11b5 100644 --- a/catalog-rest-service/src/main/resources/json/schema/tests/basic.json +++ b/catalog-rest-service/src/main/resources/json/schema/tests/basic.json @@ -13,7 +13,7 @@ "description": "Data one which profile is taken.", "$ref": "../type/basic.json#/definitions/timestamp" }, - "status": { + "testCaseStatus": { "description": "Status of Test Case run.", "javaType": "org.openmetadata.catalog.tests.type.TestCaseStatus", "type": "string", diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json b/catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json index 021dd0a63e6..574dee2b685 100644 --- a/catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json +++ b/catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json @@ -9,6 +9,7 @@ "columnTestCase": { "description": "Column Test Case.", "type": "object", + "javaType": "org.openmetadata.catalog.tests.ColumnTestCase", "properties": { "config": { "oneOf": [ @@ -35,7 +36,7 @@ } ] }, - "testType": { + "columnTestType": { "enum": [ "columnValuesToBeUnique", "columnValuesToBeNotNull", @@ -94,11 +95,6 @@ "updatedBy": { "description": "User who made the update.", "type": "string" - }, - "deleted": { - "description": "When `true` indicates the entity has been soft deleted.", - "type": "boolean", - "default": false } }, "required": ["name", "column", "testCase"], diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json index 3f2e20154cc..a139d860ac1 100644 --- a/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json +++ b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/tests/tableColumnCountToEqual.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "TableRowCountToEqual", + "title": "TableColumnCountToEqual", "description": "This scheam defines the test TableColumnCountToEqual. Test the number of columns equal to a value.", "type": "object", "javaType": "org.openmetadata.catalog.tests.table.TableColumnCountToEqual", diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/table/tableRowCountToBeBetween.json b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableRowCountToBeBetween.json index a5fa0d12dc2..dc732a2df54 100644 --- a/catalog-rest-service/src/main/resources/json/schema/tests/table/tableRowCountToBeBetween.json +++ b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableRowCountToBeBetween.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/tests/tableRowCountToBeBetween.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "TableRowCountToEqual", + "title": "TableRowCountToBeBetween", "description": "This scheam defines the test TableRowCountToBeBetween. Test the number of rows to between to two values.", "type": "object", "javaType": "org.openmetadata.catalog.tests.table.TableRowCountToBeBetween", diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/tableTest.json b/catalog-rest-service/src/main/resources/json/schema/tests/tableTest.json index e434f7df0ed..6b242bd76b6 100644 --- a/catalog-rest-service/src/main/resources/json/schema/tests/tableTest.json +++ b/catalog-rest-service/src/main/resources/json/schema/tests/tableTest.json @@ -9,6 +9,7 @@ "tableTestCase": { "description": "Table Test Case.", "type": "object", + "javaType": "org.openmetadata.catalog.tests.TableTestCase", "properties": { "config": { "oneOf": [ @@ -23,7 +24,7 @@ } ] }, - "testType": { + "tableTestType": { "enum": [ "tableRowCountToEqual", "tableRowCountToBeBetween", @@ -49,11 +50,7 @@ "description": "Description of the testcase.", "type": "string" }, - "tableName": { - "description": "Table Name for which this test applies.", - "type": "string" - }, - "tableTestCase": { + "testCase": { "$ref": "#/definitions/tableTestCase" }, "executionFrequency": { @@ -78,13 +75,8 @@ "updatedBy": { "description": "User who made the update.", "type": "string" - }, - "deleted": { - "description": "When `true` indicates the entity has been soft deleted.", - "type": "boolean", - "default": false } }, - "required": ["name", "testCase", "tableName"], + "required": ["name", "testCase"], "additionalProperties": false } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java index c1e750c3f81..d7c074c5408 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java @@ -74,6 +74,8 @@ import org.openmetadata.catalog.CatalogApplicationTest; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.api.data.CreateLocation; import org.openmetadata.catalog.api.data.CreateTable; +import org.openmetadata.catalog.api.tests.CreateColumnTest; +import org.openmetadata.catalog.api.tests.CreateTableTest; import org.openmetadata.catalog.entity.data.Database; import org.openmetadata.catalog.entity.data.Location; import org.openmetadata.catalog.entity.data.Table; @@ -1034,116 +1036,145 @@ public class TableResourceTest extends EntityResourceTest { } @Test - void put_tableColumnTests_200(TestInfo test) throws IOException { + void createUpdateDelete_tableColumnTests_200(TestInfo test) throws IOException { Table table = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS); TableRowCountToEqual tableRowCountToEqual = new TableRowCountToEqual().withValue(100); TableTestCase tableTestCase = new TableTestCase() - .withTestType(TableTestCase.TestType.TABLE_ROW_COUNT_TO_EQUAL) + .withTableTestType(TableTestCase.TableTestType.TABLE_ROW_COUNT_TO_EQUAL) .withConfig(tableRowCountToEqual); - TableTest tableTest = - new TableTest() - .withName("test1") - .withTableTestCase(tableTestCase) - .withExecutionFrequency(TestCaseExecutionFrequency.Hourly); - Table putResponse = putTableTest(table.getId(), tableTest, ADMIN_AUTH_HEADERS); - verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(tableTest)); + CreateTableTest createTableTest = + new CreateTableTest().withTestCase(tableTestCase).withExecutionFrequency(TestCaseExecutionFrequency.Hourly); + Table putResponse = putTableTest(table.getId(), createTableTest, ADMIN_AUTH_HEADERS); + verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(createTableTest)); table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS); - verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest)); + verifyTableTest(table.getName(), table.getTableTests(), List.of(createTableTest)); // Add result to tableTest TestCaseResult testCaseResult1 = new TestCaseResult() .withResult("Rows equal to 100") - .withStatus(TestCaseStatus.Success) + .withTestCaseStatus(TestCaseStatus.Success) .withSampleData("Rows == 100") .withExecutionTime(100L); - tableTest.setResults(List.of(testCaseResult1)); - tableTest.setId(table.getTableTests().get(0).getId()); - putResponse = putTableTest(table.getId(), tableTest, ADMIN_AUTH_HEADERS); - verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(tableTest)); + createTableTest.setResult(testCaseResult1); + putResponse = putTableTest(table.getId(), createTableTest, ADMIN_AUTH_HEADERS); + verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(createTableTest)); TestCaseResult testCaseResult2 = new TestCaseResult() .withResult("Rows equal to 100") - .withStatus(TestCaseStatus.Success) + .withTestCaseStatus(TestCaseStatus.Success) .withSampleData("Rows == 100") .withExecutionTime(100L); - tableTest.setResults(List.of(testCaseResult2)); - tableTest.setId(table.getTableTests().get(0).getId()); - + createTableTest.setResult(testCaseResult2); table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS); - verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest)); + verifyTableTest(table.getName(), table.getTableTests(), List.of(createTableTest)); TableRowCountToBeBetween tableRowCountToBeBetween = new TableRowCountToBeBetween().withMinValue(100).withMaxValue(1000); TableTestCase tableTestCase1 = new TableTestCase() - .withTestType(TableTestCase.TestType.TABLE_ROW_COUNT_TO_BE_BETWEEN) + .withTableTestType(TableTestCase.TableTestType.TABLE_ROW_COUNT_TO_BE_BETWEEN) .withConfig(tableRowCountToBeBetween); - TableTest tableTest1 = new TableTest().withName("column_value_to_be_unique").withTableTestCase(tableTestCase1); - putResponse = putTableTest(table.getId(), tableTest1, ADMIN_AUTH_HEADERS); - verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(tableTest, tableTest1)); + CreateTableTest createTableTest1 = new CreateTableTest().withTestCase(tableTestCase1); + putResponse = putTableTest(table.getId(), createTableTest1, ADMIN_AUTH_HEADERS); + // returns the current test thats updated or created + verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(createTableTest1)); table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS); - verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest, tableTest1)); + verifyTableTest(table.getName(), table.getTableTests(), List.of(createTableTest, createTableTest1)); + + // update the test case + tableRowCountToBeBetween = new TableRowCountToBeBetween().withMaxValue(10).withMaxValue(100); + tableTestCase1.withConfig(tableRowCountToBeBetween); + putResponse = putTableTest(table.getId(), createTableTest1, ADMIN_AUTH_HEADERS); + // returns the current test thats updated or created + verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(createTableTest1)); Column c1 = table.getColumns().get(0); ColumnValueLengthsToBeBetween columnValueLengthsToBeBetween = new ColumnValueLengthsToBeBetween().withMaxValue(100).withMinValue(10); ColumnTestCase columnTestCase = new ColumnTestCase() - .withTestType(ColumnTestCase.TestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN) + .withColumnTestType(ColumnTestCase.ColumnTestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN) .withConfig(columnValueLengthsToBeBetween); - ColumnTest columnTest = - new ColumnTest() + CreateColumnTest createColumnTest = + new CreateColumnTest() .withColumnName(c1.getName()) - .withName("test") .withTestCase(columnTestCase) .withExecutionFrequency(TestCaseExecutionFrequency.Hourly); - putResponse = putColumnTest(table.getId(), columnTest, ADMIN_AUTH_HEADERS); - verifyColumnTest(putResponse, c1, List.of(columnTest)); + putResponse = putColumnTest(table.getId(), createColumnTest, ADMIN_AUTH_HEADERS); + verifyColumnTest(putResponse, c1, List.of(createColumnTest)); table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS); - verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest, tableTest1)); - verifyColumnTest(table, c1, List.of(columnTest)); + verifyTableTest(table.getName(), table.getTableTests(), List.of(createTableTest, createTableTest1)); + verifyColumnTest(table, c1, List.of(createColumnTest)); // Add result to columnTest TestCaseResult colTestCaseResult = new TestCaseResult() .withResult("min is > 100 and max < 1000") - .withStatus(TestCaseStatus.Success) + .withTestCaseStatus(TestCaseStatus.Success) .withSampleData("minValue is 100 and maxValue is 1000") .withExecutionTime(100L); - columnTest.setResults(List.of(colTestCaseResult)); - putResponse = putColumnTest(table.getId(), columnTest, ADMIN_AUTH_HEADERS); - verifyColumnTest(putResponse, c1, List.of(columnTest)); + createColumnTest.setResult(colTestCaseResult); + putResponse = putColumnTest(table.getId(), createColumnTest, ADMIN_AUTH_HEADERS); + verifyColumnTest(putResponse, c1, List.of(createColumnTest)); ColumnValuesMissingCountToBeEqual columnValuesMissingCountToBeEqual = new ColumnValuesMissingCountToBeEqual().withMissingCountValue(10); ColumnTestCase columnTestCase1 = new ColumnTestCase() - .withTestType(ColumnTestCase.TestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL) + .withColumnTestType(ColumnTestCase.ColumnTestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL) .withConfig(columnValuesMissingCountToBeEqual); - ColumnTest columnTest1 = - new ColumnTest() + CreateColumnTest createColumnTest1 = + new CreateColumnTest() .withColumnName(c1.getName()) - .withName("test") .withTestCase(columnTestCase1) .withExecutionFrequency(TestCaseExecutionFrequency.Hourly); - putResponse = putColumnTest(table.getId(), columnTest1, ADMIN_AUTH_HEADERS); - verifyColumnTest(putResponse, c1, List.of(columnTest, columnTest1)); + putResponse = putColumnTest(table.getId(), createColumnTest1, ADMIN_AUTH_HEADERS); + verifyColumnTest(putResponse, c1, List.of(createColumnTest1)); + + // update the test config + columnValuesMissingCountToBeEqual = new ColumnValuesMissingCountToBeEqual().withMissingCountValue(100); + columnTestCase1 = + new ColumnTestCase() + .withColumnTestType(ColumnTestCase.ColumnTestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL) + .withConfig(columnValuesMissingCountToBeEqual); + createColumnTest1 = + new CreateColumnTest() + .withColumnName(c1.getName()) + .withTestCase(columnTestCase1) + .withExecutionFrequency(TestCaseExecutionFrequency.Hourly); + putResponse = putColumnTest(table.getId(), createColumnTest1, ADMIN_AUTH_HEADERS); + verifyColumnTest(putResponse, c1, List.of(createColumnTest1)); // Add result to columnTest TestCaseResult colTestCaseResult1 = new TestCaseResult() .withResult("min is > 100 and max < 1000") - .withStatus(TestCaseStatus.Success) + .withTestCaseStatus(TestCaseStatus.Success) .withSampleData("minValue is 100 and maxValue is 1000") .withExecutionTime(100L); - columnTest.setResults(List.of(colTestCaseResult1)); - putResponse = putColumnTest(table.getId(), columnTest, ADMIN_AUTH_HEADERS); - columnTest.setResults(List.of(colTestCaseResult, colTestCaseResult1)); - verifyColumnTest(putResponse, c1, List.of(columnTest, columnTest1)); + createColumnTest.setResult(colTestCaseResult1); + putResponse = putColumnTest(table.getId(), createColumnTest, ADMIN_AUTH_HEADERS); + createColumnTest.setResult(colTestCaseResult1); + verifyColumnTest(putResponse, c1, List.of(createColumnTest)); + + table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS); + verifyColumnTest(table, c1, List.of(createColumnTest, createColumnTest1)); + + // delete the table test case + putResponse = + deleteTableTest( + table.getId(), createTableTest1.getTestCase().getTableTestType().toString(), ADMIN_AUTH_HEADERS); + table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS); + verifyTableTest(table.getName(), table.getTableTests(), List.of(createTableTest)); + + // delete column test case + deleteColumnTest(table.getId(), c1.getName(), columnTestCase1.getColumnTestType().toString(), ADMIN_AUTH_HEADERS); + table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS); + verifyColumnTest(table, c1, List.of(createColumnTest)); } @Test @@ -1639,18 +1670,32 @@ public class TableResourceTest extends EntityResourceTest { return TestUtils.put(target, dataModel, Table.class, OK, authHeaders); } - public static Table putTableTest(UUID tableId, TableTest data, Map authHeaders) + public static Table putTableTest(UUID tableId, CreateTableTest data, Map authHeaders) throws HttpResponseException { WebTarget target = CatalogApplicationTest.getResource("tables/" + tableId + "/tableTest"); return TestUtils.put(target, data, Table.class, OK, authHeaders); } - public static Table putColumnTest(UUID tableId, ColumnTest data, Map authHeaders) + public static Table deleteTableTest(UUID tableId, String tableTestType, Map authHeaders) + throws HttpResponseException { + WebTarget target = CatalogApplicationTest.getResource("tables/" + tableId + "/tableTest/" + tableTestType); + return TestUtils.delete(target, Table.class, authHeaders); + } + + public static Table putColumnTest(UUID tableId, CreateColumnTest data, Map authHeaders) throws HttpResponseException { WebTarget target = CatalogApplicationTest.getResource("tables/" + tableId + "/columnTest"); return TestUtils.put(target, data, Table.class, OK, authHeaders); } + public static Table deleteColumnTest( + UUID tableId, String columnName, String columnTestType, Map authHeaders) + throws HttpResponseException { + WebTarget target = + CatalogApplicationTest.getResource("tables/" + tableId + "/columnTest/" + columnName + "/" + columnTestType); + return TestUtils.delete(target, Table.class, authHeaders); + } + private static int getTagUsageCount(String tagFQN, Map authHeaders) throws HttpResponseException { return TagResourceTest.getTag(tagFQN, "usageCount", authHeaders).getUsageCount(); } @@ -1673,45 +1718,48 @@ public class TableResourceTest extends EntityResourceTest { } } - private void verifyTableTest(String tableName, List actualTests, List expectedTests) { - assertEquals(actualTests.size(), expectedTests.size()); + private void verifyTableTest(String tableName, List actualTests, List expectedTests) + throws IOException { + assertEquals(expectedTests.size(), actualTests.size()); Map tableTestMap = new HashMap<>(); for (TableTest test : actualTests) { tableTestMap.put(test.getName(), test); } - for (TableTest test : expectedTests) { + for (CreateTableTest test : expectedTests) { // passed in test name will be overridden in backend - String expectedTestName = tableName + "." + test.getTableTestCase().getTestType().toString(); + String expectedTestName = tableName + "." + test.getTestCase().getTableTestType().toString(); TableTest storedTest = tableTestMap.get(expectedTestName); assertNotNull(storedTest); assertEquals(expectedTestName, storedTest.getName()); assertEquals(test.getDescription(), storedTest.getDescription()); assertEquals(test.getExecutionFrequency(), storedTest.getExecutionFrequency()); assertEquals(test.getOwner(), storedTest.getOwner()); - verifyTableTestCase(test.getTableTestCase(), storedTest.getTableTestCase()); - verifyTestCaseResults(test.getResults(), storedTest.getResults()); + verifyTableTestCase(test.getTestCase(), storedTest.getTestCase()); + if (test.getResult() != null && storedTest.getResults().size() > 0) { + verifyTestCaseResults(test.getResult(), storedTest.getResults()); + } } } private void verifyTableTestCase(TableTestCase expected, TableTestCase actual) { - assertEquals(expected.getTestType(), actual.getTestType()); - if (expected.getTestType() == TableTestCase.TestType.TABLE_COLUMN_COUNT_TO_EQUAL) { + assertEquals(expected.getTableTestType(), actual.getTableTestType()); + if (expected.getTableTestType() == TableTestCase.TableTestType.TABLE_COLUMN_COUNT_TO_EQUAL) { TableColumnCountToEqual expectedTest = (TableColumnCountToEqual) expected.getConfig(); TableColumnCountToEqual actualTest = JsonUtils.convertValue(actual.getConfig(), TableColumnCountToEqual.class); assertEquals(expectedTest.getValue(), actualTest.getValue()); - } else if (expected.getTestType() == TableTestCase.TestType.TABLE_ROW_COUNT_TO_BE_BETWEEN) { + } else if (expected.getTableTestType() == TableTestCase.TableTestType.TABLE_ROW_COUNT_TO_BE_BETWEEN) { TableRowCountToBeBetween expectedTest = (TableRowCountToBeBetween) expected.getConfig(); TableRowCountToBeBetween actualTest = JsonUtils.convertValue(actual.getConfig(), TableRowCountToBeBetween.class); assertEquals(expectedTest.getMaxValue(), actualTest.getMaxValue()); assertEquals(expectedTest.getMinValue(), actualTest.getMinValue()); - } else if (expected.getTestType() == TableTestCase.TestType.TABLE_ROW_COUNT_TO_EQUAL) { + } else if (expected.getTableTestType() == TableTestCase.TableTestType.TABLE_ROW_COUNT_TO_EQUAL) { TableRowCountToEqual expectedTest = (TableRowCountToEqual) expected.getConfig(); TableRowCountToEqual actualTest = JsonUtils.convertValue(actual.getConfig(), TableRowCountToEqual.class); assertEquals(expectedTest.getValue(), actualTest.getValue()); } } - private void verifyColumnTest(Table table, Column column, List expectedTests) { + private void verifyColumnTest(Table table, Column column, List expectedTests) throws IOException { List actualTests = new ArrayList<>(); for (Column c : table.getColumns()) { if (c.getName().equals(column.getName())) { @@ -1725,9 +1773,9 @@ public class TableResourceTest extends EntityResourceTest { columnTestMap.put(test.getName(), test); } - for (ColumnTest test : expectedTests) { + for (CreateColumnTest test : expectedTests) { // passed in test name will be overridden in backend - String expectedTestName = column.getName() + "." + test.getTestCase().getTestType().toString(); + String expectedTestName = column.getName() + "." + test.getTestCase().getColumnTestType().toString(); ColumnTest storedTest = columnTestMap.get(expectedTestName); assertNotNull(storedTest); assertEquals(expectedTestName, storedTest.getName()); @@ -1735,31 +1783,33 @@ public class TableResourceTest extends EntityResourceTest { assertEquals(test.getExecutionFrequency(), storedTest.getExecutionFrequency()); assertEquals(test.getOwner(), storedTest.getOwner()); verifyColumnTestCase(test.getTestCase(), storedTest.getTestCase()); - verifyTestCaseResults(test.getResults(), storedTest.getResults()); + if (test.getResult() != null) { + verifyTestCaseResults(test.getResult(), storedTest.getResults()); + } } } private void verifyColumnTestCase(ColumnTestCase expected, ColumnTestCase actual) { - assertEquals(expected.getTestType(), actual.getTestType()); - if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_TO_BE_UNIQUE) { + assertEquals(expected.getColumnTestType(), actual.getColumnTestType()); + if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUES_TO_BE_UNIQUE) { ColumnValuesToBeUnique expectedTest = (ColumnValuesToBeUnique) expected.getConfig(); ColumnValuesToBeUnique actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesToBeUnique.class); assertEquals(expectedTest, actualTest); - } else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_TO_BE_NOT_NULL) { + } else if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUES_TO_BE_NOT_NULL) { ColumnValuesToBeNotNull expectedTest = (ColumnValuesToBeNotNull) expected.getConfig(); ColumnValuesToBeNotNull actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesToBeNotNull.class); assertEquals(expectedTest, actualTest); - } else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_TO_MATCH_REGEX) { + } else if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUES_TO_MATCH_REGEX) { ColumnValuesToMatchRegex expectedTest = (ColumnValuesToMatchRegex) expected.getConfig(); ColumnValuesToMatchRegex actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesToMatchRegex.class); assertEquals(expectedTest.getRegex(), actualTest.getRegex()); - } else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN) { + } else if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN) { ColumnValueLengthsToBeBetween expectedTest = (ColumnValueLengthsToBeBetween) expected.getConfig(); ColumnValueLengthsToBeBetween actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValueLengthsToBeBetween.class); assertEquals(expectedTest.getMaxValue(), actualTest.getMaxValue()); assertEquals(expectedTest.getMinValue(), actualTest.getMinValue()); - } else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL) { + } else if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL) { ColumnValuesMissingCountToBeEqual expectedTest = (ColumnValuesMissingCountToBeEqual) expected.getConfig(); ColumnValuesMissingCountToBeEqual actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesMissingCountToBeEqual.class); @@ -1768,20 +1818,17 @@ public class TableResourceTest extends EntityResourceTest { } } - private void verifyTestCaseResults(List expected, List actual) { - assertEquals(expected.size(), actual.size()); + private void verifyTestCaseResults(TestCaseResult expected, List actual) throws IOException { Map actualResultMap = new HashMap<>(); for (Object a : actual) { TestCaseResult result = JsonUtils.convertValue(a, TestCaseResult.class); actualResultMap.put(result.getExecutionTime(), result); } - for (Object e : expected) { - TestCaseResult result = JsonUtils.convertValue(e, TestCaseResult.class); - TestCaseResult actualResult = actualResultMap.get(result.getExecutionTime()); - assertNotNull(actualResult); - assertEquals(result.getResult(), actualResult.getResult()); - assertEquals(result.getSampleData(), actualResult.getSampleData()); - } + TestCaseResult result = JsonUtils.convertValue(expected, TestCaseResult.class); + TestCaseResult actualResult = actualResultMap.get(result.getExecutionTime()); + assertNotNull(actualResult); + assertEquals(result.getResult(), actualResult.getResult()); + assertEquals(result.getSampleData(), actualResult.getSampleData()); } @Override