Fix #2989: Add delete apis for tests (#3020)

This commit is contained in:
Sriharsha Chintalapani 2022-03-01 09:43:57 -08:00 committed by GitHub
parent 282c1f68ea
commit c65cdbb6ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 372 additions and 132 deletions

View File

@ -50,6 +50,7 @@ import org.openmetadata.catalog.entity.data.Database;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.entity.services.DatabaseService;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.resources.databases.TableResource;
import org.openmetadata.catalog.tests.ColumnTest;
import org.openmetadata.catalog.tests.TableTest;
@ -260,27 +261,23 @@ public class TableRepository extends EntityRepository<Table> {
List<TableTest> storedTableTests = getTableTests(table);
// we will override any test case name passed by user/client with tableName + testType
// our assumption is there is only one instance of a test type as of now.
tableTest.setName(table.getName() + "." + tableTest.getTableTestCase().getTestType().toString());
tableTest.setName(table.getName() + "." + tableTest.getTestCase().getTableTestType().toString());
Map<String, TableTest> storedMapTableTests = new HashMap<>();
if (storedTableTests != null) {
for (TableTest t : storedTableTests) {
storedMapTableTests.put(t.getName(), t);
}
}
// new test add UUID
if (!storedMapTableTests.containsKey(tableTest.getName())) {
tableTest.setId(UUID.randomUUID());
}
// process test result
if (storedMapTableTests.containsKey(tableTest.getName())
&& tableTest.getResults() != null
&& !tableTest.getResults().isEmpty()) {
// existing test, use the previous UUID
if (storedMapTableTests.containsKey(tableTest.getName())) {
TableTest prevTableTest = storedMapTableTests.get(tableTest.getName());
List<TestCaseResult> prevTestCaseResults = prevTableTest.getResults();
List<TestCaseResult> newTestCaseResults = tableTest.getResults();
newTestCaseResults.addAll(prevTestCaseResults);
tableTest.setResults(newTestCaseResults);
tableTest.setId(prevTableTest.getId());
// process test result
if (tableTest.getResults() != null && !tableTest.getResults().isEmpty()) {
List<TestCaseResult> prevTestCaseResults = prevTableTest.getResults();
prevTestCaseResults.addAll(tableTest.getResults());
tableTest.setResults(prevTestCaseResults);
}
}
storedMapTableTests.put(tableTest.getName(), tableTest);
@ -289,7 +286,35 @@ public class TableRepository extends EntityRepository<Table> {
.entityExtensionDAO()
.insert(tableId.toString(), "table.tableTests", "tableTest", JsonUtils.pojoToJson(updatedTests));
setFields(table, Fields.EMPTY_FIELDS);
return table.withTableTests(getTableTests(table));
// return the only test instead of querying all tests and results
return table.withTableTests(List.of(tableTest));
}
@Transaction
public Table deleteTableTest(UUID tableId, String tableTestType) throws IOException, ParseException {
// Validate the request content
Table table = daoCollection.tableDAO().findEntityById(tableId);
// if ID is not passed we treat it as a new test case being added
List<TableTest> storedTableTests = getTableTests(table);
// we will override any test case name passed by user/client with tableName + testType
// our assumption is there is only one instance of a test type as of now.
String tableTestName = table.getName() + "." + tableTestType;
Map<String, TableTest> storedMapTableTests = new HashMap<>();
if (storedTableTests != null) {
for (TableTest t : storedTableTests) {
storedMapTableTests.put(t.getName(), t);
}
}
if (!storedMapTableTests.containsKey(tableTestName)) {
throw new EntityNotFoundException(String.format("Failed to find %s for %s", tableTestName, table.getName()));
}
TableTest deleteTableTest = storedMapTableTests.get(tableTestName);
storedMapTableTests.remove(tableTestName);
List<TableTest> updatedTests = new ArrayList<>(storedMapTableTests.values());
daoCollection
.entityExtensionDAO()
.insert(tableId.toString(), "table.tableTests", "tableTest", JsonUtils.pojoToJson(updatedTests));
return table.withTableTests(List.of(deleteTableTest));
}
@Transaction
@ -300,7 +325,7 @@ public class TableRepository extends EntityRepository<Table> {
validateColumn(table, columnName);
// we will override any test case name passed by user/client with columnName + testType
// our assumption is there is only one instance of a test type as of now.
columnTest.setName(columnName + "." + columnTest.getTestCase().getTestType().toString());
columnTest.setName(columnName + "." + columnTest.getTestCase().getColumnTestType().toString());
List<ColumnTest> storedColumnTests = getColumnTests(table, columnName);
Map<String, ColumnTest> storedMapColumnTests = new HashMap<>();
if (storedColumnTests != null) {
@ -309,20 +334,17 @@ public class TableRepository extends EntityRepository<Table> {
}
}
// new test, generate UUID
if (!storedMapColumnTests.containsKey(columnTest.getName())) {
columnTest.setId(UUID.randomUUID());
}
// process test result
if (storedMapColumnTests.containsKey(columnTest.getName())
&& columnTest.getResults() != null
&& !columnTest.getResults().isEmpty()) {
// existingTest use the previous UUID
if (storedMapColumnTests.containsKey(columnTest.getName())) {
ColumnTest prevColumnTest = storedMapColumnTests.get(columnTest.getName());
List<TestCaseResult> prevTestCaseResults = prevColumnTest.getResults();
List<TestCaseResult> newTestCaseResults = columnTest.getResults();
newTestCaseResults.addAll(prevTestCaseResults);
columnTest.setResults(newTestCaseResults);
columnTest.setId(prevColumnTest.getId());
// process test results
if (columnTest.getResults() != null && !columnTest.getResults().isEmpty()) {
List<TestCaseResult> prevTestCaseResults = prevColumnTest.getResults();
prevTestCaseResults.addAll(columnTest.getResults());
columnTest.setResults(prevTestCaseResults);
}
}
storedMapColumnTests.put(columnTest.getName(), columnTest);
@ -332,7 +354,48 @@ public class TableRepository extends EntityRepository<Table> {
.entityExtensionDAO()
.insert(table.getId().toString(), extension, "columnTest", JsonUtils.pojoToJson(updatedTests));
setFields(table, Fields.EMPTY_FIELDS);
getColumnTests(true, table);
// return the newly created/updated column test only
for (Column column : table.getColumns()) {
if (column.getName().equals(columnName)) {
column.setColumnTests(List.of(columnTest));
}
}
return table;
}
@Transaction
public Table deleteColumnTest(UUID tableId, String columnName, String columnTestType) throws IOException {
// Validate the request content
Table table = daoCollection.tableDAO().findEntityById(tableId);
validateColumn(table, columnName);
// we will override any test case name passed by user/client with columnName + testType
// our assumption is there is only one instance of a test type as of now.
String columnTestName = columnName + "." + columnTestType;
List<ColumnTest> storedColumnTests = getColumnTests(table, columnName);
Map<String, ColumnTest> storedMapColumnTests = new HashMap<>();
if (storedColumnTests != null) {
for (ColumnTest ct : storedColumnTests) {
storedMapColumnTests.put(ct.getName(), ct);
}
}
if (!storedMapColumnTests.containsKey(columnTestName)) {
throw new EntityNotFoundException(String.format("Failed to find %s for %s", columnTestName, table.getName()));
}
ColumnTest deleteColumnTest = storedMapColumnTests.get(columnTestName);
storedMapColumnTests.remove(columnTestName);
List<ColumnTest> updatedTests = new ArrayList<>(storedMapColumnTests.values());
String extension = "table.column." + columnName + ".tests";
daoCollection
.entityExtensionDAO()
.insert(table.getId().toString(), extension, "columnTest", JsonUtils.pojoToJson(updatedTests));
// return the newly created/updated column test only
for (Column column : table.getColumns()) {
if (column.getName().equals(columnName)) {
column.setColumnTests(List.of(deleteColumnTest));
}
}
return table;
}

View File

@ -52,6 +52,8 @@ import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateTable;
import org.openmetadata.catalog.api.tests.CreateColumnTest;
import org.openmetadata.catalog.api.tests.CreateTableTest;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.jdbi3.CollectionDAO;
import org.openmetadata.catalog.jdbi3.TableRepository;
@ -521,13 +523,29 @@ public class TableResource {
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id,
TableTest tableTest)
CreateTableTest createTableTest)
throws IOException, ParseException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
TableTest tableTest = getTableTest(securityContext, createTableTest);
Table table = dao.addTableTest(UUID.fromString(id), tableTest);
return addHref(uriInfo, table);
}
@DELETE
@Path("/{id}/tableTest/{tableTestType}")
@Operation(summary = "delete table test case", tags = "tables", description = "Delete test case from the table.")
public Table deleteTableTest(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id,
@Parameter(description = "Table Test Type", schema = @Schema(type = "string")) @PathParam("tableTestType")
String tableTestType)
throws IOException, ParseException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
Table table = dao.deleteTableTest(UUID.fromString(id), tableTestType);
return addHref(uriInfo, table);
}
@PUT
@Path("/{id}/columnTest")
@Operation(summary = "Add table test cases", tags = "tables", description = "Add test cases to the table.")
@ -535,13 +553,34 @@ public class TableResource {
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id,
ColumnTest columnTest)
CreateColumnTest createColumnTest)
throws IOException, ParseException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
ColumnTest columnTest = getColumnTest(securityContext, createColumnTest);
Table table = dao.addColumnTest(UUID.fromString(id), columnTest);
return addHref(uriInfo, table);
}
@DELETE
@Path("/{id}/columnTest/{columnName}/{columnTestType}")
@Operation(
summary = "delete column test case",
tags = "tables",
description = "Delete column test case from the table.")
public Table deleteColumnTest(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id,
@Parameter(description = "column of the table", schema = @Schema(type = "string")) @PathParam("columnName")
String columnName,
@Parameter(description = "column Test Type", schema = @Schema(type = "string")) @PathParam("columnTestType")
String columnTestType)
throws IOException, ParseException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
Table table = dao.deleteColumnTest(UUID.fromString(id), columnName, columnTestType);
return addHref(uriInfo, table);
}
@DELETE
@Path("/{id}/followers/{userId}")
@Operation(
@ -598,4 +637,29 @@ public class TableResource {
.withUpdatedAt(System.currentTimeMillis())
.withDatabase(create.getDatabase());
}
private TableTest getTableTest(SecurityContext securityContext, CreateTableTest create) {
return new TableTest()
.withId(UUID.randomUUID())
.withDescription(create.getDescription())
.withTestCase(create.getTestCase())
.withOwner(create.getOwner())
.withExecutionFrequency(create.getExecutionFrequency())
.withResults(create.getResult() != null ? List.of(create.getResult()) : new ArrayList<>())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(System.currentTimeMillis());
}
private ColumnTest getColumnTest(SecurityContext securityContext, CreateColumnTest create) {
return new ColumnTest()
.withId(UUID.randomUUID())
.withDescription(create.getDescription())
.withTestCase(create.getTestCase())
.withColumnName(create.getColumnName())
.withOwner(create.getOwner())
.withExecutionFrequency(create.getExecutionFrequency())
.withResults(create.getResult() != null ? List.of(create.getResult()) : new ArrayList<>())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(System.currentTimeMillis());
}
}

View File

@ -0,0 +1,41 @@
{
"$id": "https://open-metadata.org/schema/api/tests/columnTest.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "CreateColumnTestRequest",
"description": "ColumnTest is a test definition to capture data quality tests against tables and columns.",
"type": "object",
"properties": {
"description": {
"description": "Description of the testcase.",
"type": "string"
},
"columnName": {
"description": "Name of the column in a table.",
"type": "string"
},
"testCase": {
"$ref": "../../tests/columnTest.json#/definitions/columnTestCase"
},
"executionFrequency": {
"$ref": "../../tests/basic.json#/definitions/testCaseExecutionFrequency"
},
"result": {
"$ref": "../../tests/basic.json#/definitions/testCaseResult"
},
"owner": {
"description": "Owner of this Pipeline.",
"$ref": "../../type/entityReference.json",
"default": null
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"updatedBy": {
"description": "User who made the update.",
"type": "string"
}
},
"required": ["columnName", "testCase"],
"additionalProperties": false
}

View File

@ -0,0 +1,37 @@
{
"$id": "https://open-metadata.org/schema/api/tests/tableTest.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "CreateTableTestRequest",
"description": "TableTest is a test definition to capture data quality tests against tables and columns.",
"type": "object",
"properties": {
"description": {
"description": "Description of the testcase.",
"type": "string"
},
"testCase": {
"$ref": "../../tests/tableTest.json#/definitions/tableTestCase"
},
"executionFrequency": {
"$ref": "../../tests/basic.json#/definitions/testCaseExecutionFrequency"
},
"result": {
"$ref": "../../tests/basic.json#/definitions/testCaseResult"
},
"owner": {
"description": "Owner of this Pipeline.",
"$ref": "../../type/entityReference.json",
"default": null
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"updatedBy": {
"description": "User who made the update.",
"type": "string"
}
},
"required": ["testCase"],
"additionalProperties": false
}

View File

@ -13,7 +13,7 @@
"description": "Data one which profile is taken.",
"$ref": "../type/basic.json#/definitions/timestamp"
},
"status": {
"testCaseStatus": {
"description": "Status of Test Case run.",
"javaType": "org.openmetadata.catalog.tests.type.TestCaseStatus",
"type": "string",

View File

@ -9,6 +9,7 @@
"columnTestCase": {
"description": "Column Test Case.",
"type": "object",
"javaType": "org.openmetadata.catalog.tests.ColumnTestCase",
"properties": {
"config": {
"oneOf": [
@ -35,7 +36,7 @@
}
]
},
"testType": {
"columnTestType": {
"enum": [
"columnValuesToBeUnique",
"columnValuesToBeNotNull",
@ -94,11 +95,6 @@
"updatedBy": {
"description": "User who made the update.",
"type": "string"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
}
},
"required": ["name", "column", "testCase"],

View File

@ -1,7 +1,7 @@
{
"$id": "https://open-metadata.org/schema/tests/tableColumnCountToEqual.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TableRowCountToEqual",
"title": "TableColumnCountToEqual",
"description": "This scheam defines the test TableColumnCountToEqual. Test the number of columns equal to a value.",
"type": "object",
"javaType": "org.openmetadata.catalog.tests.table.TableColumnCountToEqual",

View File

@ -1,7 +1,7 @@
{
"$id": "https://open-metadata.org/schema/tests/tableRowCountToBeBetween.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TableRowCountToEqual",
"title": "TableRowCountToBeBetween",
"description": "This scheam defines the test TableRowCountToBeBetween. Test the number of rows to between to two values.",
"type": "object",
"javaType": "org.openmetadata.catalog.tests.table.TableRowCountToBeBetween",

View File

@ -9,6 +9,7 @@
"tableTestCase": {
"description": "Table Test Case.",
"type": "object",
"javaType": "org.openmetadata.catalog.tests.TableTestCase",
"properties": {
"config": {
"oneOf": [
@ -23,7 +24,7 @@
}
]
},
"testType": {
"tableTestType": {
"enum": [
"tableRowCountToEqual",
"tableRowCountToBeBetween",
@ -49,11 +50,7 @@
"description": "Description of the testcase.",
"type": "string"
},
"tableName": {
"description": "Table Name for which this test applies.",
"type": "string"
},
"tableTestCase": {
"testCase": {
"$ref": "#/definitions/tableTestCase"
},
"executionFrequency": {
@ -78,13 +75,8 @@
"updatedBy": {
"description": "User who made the update.",
"type": "string"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
}
},
"required": ["name", "testCase", "tableName"],
"required": ["name", "testCase"],
"additionalProperties": false
}

View File

@ -74,6 +74,8 @@ import org.openmetadata.catalog.CatalogApplicationTest;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateLocation;
import org.openmetadata.catalog.api.data.CreateTable;
import org.openmetadata.catalog.api.tests.CreateColumnTest;
import org.openmetadata.catalog.api.tests.CreateTableTest;
import org.openmetadata.catalog.entity.data.Database;
import org.openmetadata.catalog.entity.data.Location;
import org.openmetadata.catalog.entity.data.Table;
@ -1034,116 +1036,145 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
}
@Test
void put_tableColumnTests_200(TestInfo test) throws IOException {
void createUpdateDelete_tableColumnTests_200(TestInfo test) throws IOException {
Table table = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS);
TableRowCountToEqual tableRowCountToEqual = new TableRowCountToEqual().withValue(100);
TableTestCase tableTestCase =
new TableTestCase()
.withTestType(TableTestCase.TestType.TABLE_ROW_COUNT_TO_EQUAL)
.withTableTestType(TableTestCase.TableTestType.TABLE_ROW_COUNT_TO_EQUAL)
.withConfig(tableRowCountToEqual);
TableTest tableTest =
new TableTest()
.withName("test1")
.withTableTestCase(tableTestCase)
.withExecutionFrequency(TestCaseExecutionFrequency.Hourly);
Table putResponse = putTableTest(table.getId(), tableTest, ADMIN_AUTH_HEADERS);
verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(tableTest));
CreateTableTest createTableTest =
new CreateTableTest().withTestCase(tableTestCase).withExecutionFrequency(TestCaseExecutionFrequency.Hourly);
Table putResponse = putTableTest(table.getId(), createTableTest, ADMIN_AUTH_HEADERS);
verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(createTableTest));
table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS);
verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest));
verifyTableTest(table.getName(), table.getTableTests(), List.of(createTableTest));
// Add result to tableTest
TestCaseResult testCaseResult1 =
new TestCaseResult()
.withResult("Rows equal to 100")
.withStatus(TestCaseStatus.Success)
.withTestCaseStatus(TestCaseStatus.Success)
.withSampleData("Rows == 100")
.withExecutionTime(100L);
tableTest.setResults(List.of(testCaseResult1));
tableTest.setId(table.getTableTests().get(0).getId());
putResponse = putTableTest(table.getId(), tableTest, ADMIN_AUTH_HEADERS);
verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(tableTest));
createTableTest.setResult(testCaseResult1);
putResponse = putTableTest(table.getId(), createTableTest, ADMIN_AUTH_HEADERS);
verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(createTableTest));
TestCaseResult testCaseResult2 =
new TestCaseResult()
.withResult("Rows equal to 100")
.withStatus(TestCaseStatus.Success)
.withTestCaseStatus(TestCaseStatus.Success)
.withSampleData("Rows == 100")
.withExecutionTime(100L);
tableTest.setResults(List.of(testCaseResult2));
tableTest.setId(table.getTableTests().get(0).getId());
createTableTest.setResult(testCaseResult2);
table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS);
verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest));
verifyTableTest(table.getName(), table.getTableTests(), List.of(createTableTest));
TableRowCountToBeBetween tableRowCountToBeBetween =
new TableRowCountToBeBetween().withMinValue(100).withMaxValue(1000);
TableTestCase tableTestCase1 =
new TableTestCase()
.withTestType(TableTestCase.TestType.TABLE_ROW_COUNT_TO_BE_BETWEEN)
.withTableTestType(TableTestCase.TableTestType.TABLE_ROW_COUNT_TO_BE_BETWEEN)
.withConfig(tableRowCountToBeBetween);
TableTest tableTest1 = new TableTest().withName("column_value_to_be_unique").withTableTestCase(tableTestCase1);
putResponse = putTableTest(table.getId(), tableTest1, ADMIN_AUTH_HEADERS);
verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(tableTest, tableTest1));
CreateTableTest createTableTest1 = new CreateTableTest().withTestCase(tableTestCase1);
putResponse = putTableTest(table.getId(), createTableTest1, ADMIN_AUTH_HEADERS);
// returns the current test thats updated or created
verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(createTableTest1));
table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS);
verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest, tableTest1));
verifyTableTest(table.getName(), table.getTableTests(), List.of(createTableTest, createTableTest1));
// update the test case
tableRowCountToBeBetween = new TableRowCountToBeBetween().withMaxValue(10).withMaxValue(100);
tableTestCase1.withConfig(tableRowCountToBeBetween);
putResponse = putTableTest(table.getId(), createTableTest1, ADMIN_AUTH_HEADERS);
// returns the current test thats updated or created
verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(createTableTest1));
Column c1 = table.getColumns().get(0);
ColumnValueLengthsToBeBetween columnValueLengthsToBeBetween =
new ColumnValueLengthsToBeBetween().withMaxValue(100).withMinValue(10);
ColumnTestCase columnTestCase =
new ColumnTestCase()
.withTestType(ColumnTestCase.TestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN)
.withColumnTestType(ColumnTestCase.ColumnTestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN)
.withConfig(columnValueLengthsToBeBetween);
ColumnTest columnTest =
new ColumnTest()
CreateColumnTest createColumnTest =
new CreateColumnTest()
.withColumnName(c1.getName())
.withName("test")
.withTestCase(columnTestCase)
.withExecutionFrequency(TestCaseExecutionFrequency.Hourly);
putResponse = putColumnTest(table.getId(), columnTest, ADMIN_AUTH_HEADERS);
verifyColumnTest(putResponse, c1, List.of(columnTest));
putResponse = putColumnTest(table.getId(), createColumnTest, ADMIN_AUTH_HEADERS);
verifyColumnTest(putResponse, c1, List.of(createColumnTest));
table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS);
verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest, tableTest1));
verifyColumnTest(table, c1, List.of(columnTest));
verifyTableTest(table.getName(), table.getTableTests(), List.of(createTableTest, createTableTest1));
verifyColumnTest(table, c1, List.of(createColumnTest));
// Add result to columnTest
TestCaseResult colTestCaseResult =
new TestCaseResult()
.withResult("min is > 100 and max < 1000")
.withStatus(TestCaseStatus.Success)
.withTestCaseStatus(TestCaseStatus.Success)
.withSampleData("minValue is 100 and maxValue is 1000")
.withExecutionTime(100L);
columnTest.setResults(List.of(colTestCaseResult));
putResponse = putColumnTest(table.getId(), columnTest, ADMIN_AUTH_HEADERS);
verifyColumnTest(putResponse, c1, List.of(columnTest));
createColumnTest.setResult(colTestCaseResult);
putResponse = putColumnTest(table.getId(), createColumnTest, ADMIN_AUTH_HEADERS);
verifyColumnTest(putResponse, c1, List.of(createColumnTest));
ColumnValuesMissingCountToBeEqual columnValuesMissingCountToBeEqual =
new ColumnValuesMissingCountToBeEqual().withMissingCountValue(10);
ColumnTestCase columnTestCase1 =
new ColumnTestCase()
.withTestType(ColumnTestCase.TestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL)
.withColumnTestType(ColumnTestCase.ColumnTestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL)
.withConfig(columnValuesMissingCountToBeEqual);
ColumnTest columnTest1 =
new ColumnTest()
CreateColumnTest createColumnTest1 =
new CreateColumnTest()
.withColumnName(c1.getName())
.withName("test")
.withTestCase(columnTestCase1)
.withExecutionFrequency(TestCaseExecutionFrequency.Hourly);
putResponse = putColumnTest(table.getId(), columnTest1, ADMIN_AUTH_HEADERS);
verifyColumnTest(putResponse, c1, List.of(columnTest, columnTest1));
putResponse = putColumnTest(table.getId(), createColumnTest1, ADMIN_AUTH_HEADERS);
verifyColumnTest(putResponse, c1, List.of(createColumnTest1));
// update the test config
columnValuesMissingCountToBeEqual = new ColumnValuesMissingCountToBeEqual().withMissingCountValue(100);
columnTestCase1 =
new ColumnTestCase()
.withColumnTestType(ColumnTestCase.ColumnTestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL)
.withConfig(columnValuesMissingCountToBeEqual);
createColumnTest1 =
new CreateColumnTest()
.withColumnName(c1.getName())
.withTestCase(columnTestCase1)
.withExecutionFrequency(TestCaseExecutionFrequency.Hourly);
putResponse = putColumnTest(table.getId(), createColumnTest1, ADMIN_AUTH_HEADERS);
verifyColumnTest(putResponse, c1, List.of(createColumnTest1));
// Add result to columnTest
TestCaseResult colTestCaseResult1 =
new TestCaseResult()
.withResult("min is > 100 and max < 1000")
.withStatus(TestCaseStatus.Success)
.withTestCaseStatus(TestCaseStatus.Success)
.withSampleData("minValue is 100 and maxValue is 1000")
.withExecutionTime(100L);
columnTest.setResults(List.of(colTestCaseResult1));
putResponse = putColumnTest(table.getId(), columnTest, ADMIN_AUTH_HEADERS);
columnTest.setResults(List.of(colTestCaseResult, colTestCaseResult1));
verifyColumnTest(putResponse, c1, List.of(columnTest, columnTest1));
createColumnTest.setResult(colTestCaseResult1);
putResponse = putColumnTest(table.getId(), createColumnTest, ADMIN_AUTH_HEADERS);
createColumnTest.setResult(colTestCaseResult1);
verifyColumnTest(putResponse, c1, List.of(createColumnTest));
table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS);
verifyColumnTest(table, c1, List.of(createColumnTest, createColumnTest1));
// delete the table test case
putResponse =
deleteTableTest(
table.getId(), createTableTest1.getTestCase().getTableTestType().toString(), ADMIN_AUTH_HEADERS);
table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS);
verifyTableTest(table.getName(), table.getTableTests(), List.of(createTableTest));
// delete column test case
deleteColumnTest(table.getId(), c1.getName(), columnTestCase1.getColumnTestType().toString(), ADMIN_AUTH_HEADERS);
table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS);
verifyColumnTest(table, c1, List.of(createColumnTest));
}
@Test
@ -1639,18 +1670,32 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
return TestUtils.put(target, dataModel, Table.class, OK, authHeaders);
}
public static Table putTableTest(UUID tableId, TableTest data, Map<String, String> authHeaders)
public static Table putTableTest(UUID tableId, CreateTableTest data, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("tables/" + tableId + "/tableTest");
return TestUtils.put(target, data, Table.class, OK, authHeaders);
}
public static Table putColumnTest(UUID tableId, ColumnTest data, Map<String, String> authHeaders)
public static Table deleteTableTest(UUID tableId, String tableTestType, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("tables/" + tableId + "/tableTest/" + tableTestType);
return TestUtils.delete(target, Table.class, authHeaders);
}
public static Table putColumnTest(UUID tableId, CreateColumnTest data, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("tables/" + tableId + "/columnTest");
return TestUtils.put(target, data, Table.class, OK, authHeaders);
}
public static Table deleteColumnTest(
UUID tableId, String columnName, String columnTestType, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target =
CatalogApplicationTest.getResource("tables/" + tableId + "/columnTest/" + columnName + "/" + columnTestType);
return TestUtils.delete(target, Table.class, authHeaders);
}
private static int getTagUsageCount(String tagFQN, Map<String, String> authHeaders) throws HttpResponseException {
return TagResourceTest.getTag(tagFQN, "usageCount", authHeaders).getUsageCount();
}
@ -1673,45 +1718,48 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
}
}
private void verifyTableTest(String tableName, List<TableTest> actualTests, List<TableTest> expectedTests) {
assertEquals(actualTests.size(), expectedTests.size());
private void verifyTableTest(String tableName, List<TableTest> actualTests, List<CreateTableTest> expectedTests)
throws IOException {
assertEquals(expectedTests.size(), actualTests.size());
Map<String, TableTest> tableTestMap = new HashMap<>();
for (TableTest test : actualTests) {
tableTestMap.put(test.getName(), test);
}
for (TableTest test : expectedTests) {
for (CreateTableTest test : expectedTests) {
// passed in test name will be overridden in backend
String expectedTestName = tableName + "." + test.getTableTestCase().getTestType().toString();
String expectedTestName = tableName + "." + test.getTestCase().getTableTestType().toString();
TableTest storedTest = tableTestMap.get(expectedTestName);
assertNotNull(storedTest);
assertEquals(expectedTestName, storedTest.getName());
assertEquals(test.getDescription(), storedTest.getDescription());
assertEquals(test.getExecutionFrequency(), storedTest.getExecutionFrequency());
assertEquals(test.getOwner(), storedTest.getOwner());
verifyTableTestCase(test.getTableTestCase(), storedTest.getTableTestCase());
verifyTestCaseResults(test.getResults(), storedTest.getResults());
verifyTableTestCase(test.getTestCase(), storedTest.getTestCase());
if (test.getResult() != null && storedTest.getResults().size() > 0) {
verifyTestCaseResults(test.getResult(), storedTest.getResults());
}
}
}
private void verifyTableTestCase(TableTestCase expected, TableTestCase actual) {
assertEquals(expected.getTestType(), actual.getTestType());
if (expected.getTestType() == TableTestCase.TestType.TABLE_COLUMN_COUNT_TO_EQUAL) {
assertEquals(expected.getTableTestType(), actual.getTableTestType());
if (expected.getTableTestType() == TableTestCase.TableTestType.TABLE_COLUMN_COUNT_TO_EQUAL) {
TableColumnCountToEqual expectedTest = (TableColumnCountToEqual) expected.getConfig();
TableColumnCountToEqual actualTest = JsonUtils.convertValue(actual.getConfig(), TableColumnCountToEqual.class);
assertEquals(expectedTest.getValue(), actualTest.getValue());
} else if (expected.getTestType() == TableTestCase.TestType.TABLE_ROW_COUNT_TO_BE_BETWEEN) {
} else if (expected.getTableTestType() == TableTestCase.TableTestType.TABLE_ROW_COUNT_TO_BE_BETWEEN) {
TableRowCountToBeBetween expectedTest = (TableRowCountToBeBetween) expected.getConfig();
TableRowCountToBeBetween actualTest = JsonUtils.convertValue(actual.getConfig(), TableRowCountToBeBetween.class);
assertEquals(expectedTest.getMaxValue(), actualTest.getMaxValue());
assertEquals(expectedTest.getMinValue(), actualTest.getMinValue());
} else if (expected.getTestType() == TableTestCase.TestType.TABLE_ROW_COUNT_TO_EQUAL) {
} else if (expected.getTableTestType() == TableTestCase.TableTestType.TABLE_ROW_COUNT_TO_EQUAL) {
TableRowCountToEqual expectedTest = (TableRowCountToEqual) expected.getConfig();
TableRowCountToEqual actualTest = JsonUtils.convertValue(actual.getConfig(), TableRowCountToEqual.class);
assertEquals(expectedTest.getValue(), actualTest.getValue());
}
}
private void verifyColumnTest(Table table, Column column, List<ColumnTest> expectedTests) {
private void verifyColumnTest(Table table, Column column, List<CreateColumnTest> expectedTests) throws IOException {
List<ColumnTest> actualTests = new ArrayList<>();
for (Column c : table.getColumns()) {
if (c.getName().equals(column.getName())) {
@ -1725,9 +1773,9 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
columnTestMap.put(test.getName(), test);
}
for (ColumnTest test : expectedTests) {
for (CreateColumnTest test : expectedTests) {
// passed in test name will be overridden in backend
String expectedTestName = column.getName() + "." + test.getTestCase().getTestType().toString();
String expectedTestName = column.getName() + "." + test.getTestCase().getColumnTestType().toString();
ColumnTest storedTest = columnTestMap.get(expectedTestName);
assertNotNull(storedTest);
assertEquals(expectedTestName, storedTest.getName());
@ -1735,31 +1783,33 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
assertEquals(test.getExecutionFrequency(), storedTest.getExecutionFrequency());
assertEquals(test.getOwner(), storedTest.getOwner());
verifyColumnTestCase(test.getTestCase(), storedTest.getTestCase());
verifyTestCaseResults(test.getResults(), storedTest.getResults());
if (test.getResult() != null) {
verifyTestCaseResults(test.getResult(), storedTest.getResults());
}
}
}
private void verifyColumnTestCase(ColumnTestCase expected, ColumnTestCase actual) {
assertEquals(expected.getTestType(), actual.getTestType());
if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_TO_BE_UNIQUE) {
assertEquals(expected.getColumnTestType(), actual.getColumnTestType());
if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUES_TO_BE_UNIQUE) {
ColumnValuesToBeUnique expectedTest = (ColumnValuesToBeUnique) expected.getConfig();
ColumnValuesToBeUnique actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesToBeUnique.class);
assertEquals(expectedTest, actualTest);
} else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_TO_BE_NOT_NULL) {
} else if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUES_TO_BE_NOT_NULL) {
ColumnValuesToBeNotNull expectedTest = (ColumnValuesToBeNotNull) expected.getConfig();
ColumnValuesToBeNotNull actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesToBeNotNull.class);
assertEquals(expectedTest, actualTest);
} else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_TO_MATCH_REGEX) {
} else if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUES_TO_MATCH_REGEX) {
ColumnValuesToMatchRegex expectedTest = (ColumnValuesToMatchRegex) expected.getConfig();
ColumnValuesToMatchRegex actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesToMatchRegex.class);
assertEquals(expectedTest.getRegex(), actualTest.getRegex());
} else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN) {
} else if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN) {
ColumnValueLengthsToBeBetween expectedTest = (ColumnValueLengthsToBeBetween) expected.getConfig();
ColumnValueLengthsToBeBetween actualTest =
JsonUtils.convertValue(actual.getConfig(), ColumnValueLengthsToBeBetween.class);
assertEquals(expectedTest.getMaxValue(), actualTest.getMaxValue());
assertEquals(expectedTest.getMinValue(), actualTest.getMinValue());
} else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL) {
} else if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL) {
ColumnValuesMissingCountToBeEqual expectedTest = (ColumnValuesMissingCountToBeEqual) expected.getConfig();
ColumnValuesMissingCountToBeEqual actualTest =
JsonUtils.convertValue(actual.getConfig(), ColumnValuesMissingCountToBeEqual.class);
@ -1768,20 +1818,17 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
}
}
private void verifyTestCaseResults(List<TestCaseResult> expected, List<TestCaseResult> actual) {
assertEquals(expected.size(), actual.size());
private void verifyTestCaseResults(TestCaseResult expected, List<TestCaseResult> actual) throws IOException {
Map<Long, TestCaseResult> actualResultMap = new HashMap<>();
for (Object a : actual) {
TestCaseResult result = JsonUtils.convertValue(a, TestCaseResult.class);
actualResultMap.put(result.getExecutionTime(), result);
}
for (Object e : expected) {
TestCaseResult result = JsonUtils.convertValue(e, TestCaseResult.class);
TestCaseResult actualResult = actualResultMap.get(result.getExecutionTime());
assertNotNull(actualResult);
assertEquals(result.getResult(), actualResult.getResult());
assertEquals(result.getSampleData(), actualResult.getSampleData());
}
TestCaseResult result = JsonUtils.convertValue(expected, TestCaseResult.class);
TestCaseResult actualResult = actualResultMap.get(result.getExecutionTime());
assertNotNull(actualResult);
assertEquals(result.getResult(), actualResult.getResult());
assertEquals(result.getSampleData(), actualResult.getSampleData());
}
@Override