diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/events/ChangeEventHandler.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/events/ChangeEventHandler.java index 02f21a73673..877913f108c 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/events/ChangeEventHandler.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/events/ChangeEventHandler.java @@ -182,7 +182,9 @@ public class ChangeEventHandler implements EventHandler { } var entityInterface = Entity.getEntityInterface(entity); - + if (entityInterface.getChangeDescription() == null) { + return null; + } List fieldsUpdated = entityInterface.getChangeDescription().getFieldsUpdated(); List threads = new ArrayList<>(getThreads(fieldsUpdated, entity, CHANGE_TYPE.UPDATE)); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java index 7618ec7414f..1544584a38f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java @@ -49,6 +49,9 @@ import org.openmetadata.catalog.entity.data.Table; import org.openmetadata.catalog.entity.services.DatabaseService; import org.openmetadata.catalog.exception.CatalogExceptionMessage; import org.openmetadata.catalog.resources.databases.TableResource; +import org.openmetadata.catalog.tests.ColumnTest; +import org.openmetadata.catalog.tests.TableTest; +import org.openmetadata.catalog.tests.type.TestCaseResult; import org.openmetadata.catalog.type.ChangeDescription; import org.openmetadata.catalog.type.Column; import org.openmetadata.catalog.type.ColumnJoin; @@ -110,6 +113,8 @@ public class TableRepository extends EntityRepository { table.setTableProfile(fields.contains("tableProfile") ? getTableProfile(table) : null); table.setLocation(fields.contains("location") ? getLocation(table) : null); table.setTableQueries(fields.contains("tableQueries") ? getQueries(table) : null); + table.setTableTests(fields.contains("tests") ? getTableTests(table) : null); + getColumnTests(fields.contains("tests"), table); return table; } @@ -245,6 +250,90 @@ public class TableRepository extends EntityRepository
{ return table.withTableQueries(getQueries(table)); } + @Transaction + public Table addTableTest(UUID tableId, TableTest tableTest) throws IOException, ParseException { + // Validate the request content + Table table = daoCollection.tableDAO().findEntityById(tableId); + // if ID is not passed we treat it as a new test case being added + List storedTableTests = getTableTests(table); + // we will override any test case name passed by user/client with tableName + testType + // our assumption is there is only one instance of a test type as of now. + tableTest.setName(table.getName() + "." + tableTest.getTableTestCase().getTestType().toString()); + Map storedMapTableTests = new HashMap<>(); + if (storedTableTests != null) { + for (TableTest t : storedTableTests) { + storedMapTableTests.put(t.getName(), t); + } + } + // new test add UUID + if (!storedMapTableTests.containsKey(tableTest.getName())) { + tableTest.setId(UUID.randomUUID()); + } + + // process test result + if (storedMapTableTests.containsKey(tableTest.getName()) + && tableTest.getResults() != null + && !tableTest.getResults().isEmpty()) { + TableTest prevTableTest = storedMapTableTests.get(tableTest.getName()); + List prevTestCaseResults = prevTableTest.getResults(); + List newTestCaseResults = tableTest.getResults(); + newTestCaseResults.addAll(prevTestCaseResults); + tableTest.setResults(newTestCaseResults); + } + + storedMapTableTests.put(tableTest.getName(), tableTest); + List updatedTests = new ArrayList<>(storedMapTableTests.values()); + daoCollection + .entityExtensionDAO() + .insert(tableId.toString(), "table.tableTests", "tableTest", JsonUtils.pojoToJson(updatedTests)); + setFields(table, Fields.EMPTY_FIELDS); + return table.withTableTests(getTableTests(table)); + } + + @Transaction + public Table addColumnTest(UUID tableId, ColumnTest columnTest) throws IOException, ParseException { + // Validate the request content + Table table = daoCollection.tableDAO().findEntityById(tableId); + String columnName = columnTest.getColumnName(); + validateColumn(table, columnName); + // we will override any test case name passed by user/client with columnName + testType + // our assumption is there is only one instance of a test type as of now. + columnTest.setName(columnName + "." + columnTest.getTestCase().getTestType().toString()); + List storedColumnTests = getColumnTests(table, columnName); + Map storedMapColumnTests = new HashMap<>(); + if (storedColumnTests != null) { + for (ColumnTest ct : storedColumnTests) { + storedMapColumnTests.put(ct.getName(), ct); + } + } + + // new test, generate UUID + if (!storedMapColumnTests.containsKey(columnTest.getName())) { + columnTest.setId(UUID.randomUUID()); + } + + // process test result + if (storedMapColumnTests.containsKey(columnTest.getName()) + && columnTest.getResults() != null + && !columnTest.getResults().isEmpty()) { + ColumnTest prevColumnTest = storedMapColumnTests.get(columnTest.getName()); + List prevTestCaseResults = prevColumnTest.getResults(); + List newTestCaseResults = columnTest.getResults(); + newTestCaseResults.addAll(prevTestCaseResults); + columnTest.setResults(newTestCaseResults); + } + + storedMapColumnTests.put(columnTest.getName(), columnTest); + List updatedTests = new ArrayList<>(storedMapColumnTests.values()); + String extension = "table.column." + columnName + ".tests"; + daoCollection + .entityExtensionDAO() + .insert(table.getId().toString(), extension, "columnTest", JsonUtils.pojoToJson(updatedTests)); + setFields(table, Fields.EMPTY_FIELDS); + getColumnTests(true, table); + return table; + } + @Transaction public Table addDataModel(UUID tableId, DataModel dataModel) throws IOException, ParseException { Table table = daoCollection.tableDAO().findEntityById(tableId); @@ -642,6 +731,24 @@ public class TableRepository extends EntityRepository
{ return tableQueries; } + private List getTableTests(Table table) throws IOException { + return JsonUtils.readObjects( + daoCollection.entityExtensionDAO().getExtension(table.getId().toString(), "table.tableTests"), TableTest.class); + } + + private List getColumnTests(Table table, String columnName) throws IOException { + String extension = "table.column." + columnName + ".tests"; + return JsonUtils.readObjects( + daoCollection.entityExtensionDAO().getExtension(table.getId().toString(), extension), ColumnTest.class); + } + + private void getColumnTests(boolean setTests, Table table) throws IOException { + List columns = table.getColumns(); + for (Column c : Optional.ofNullable(columns).orElse(Collections.emptyList())) { + c.setColumnTests(setTests ? getColumnTests(table, c.getName()) : null); + } + } + public static class TableEntityInterface implements EntityInterface
{ private final Table entity; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java index 0090374f447..e2b1a4003cf 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java @@ -57,6 +57,8 @@ import org.openmetadata.catalog.jdbi3.TableRepository; import org.openmetadata.catalog.resources.Collection; import org.openmetadata.catalog.security.Authorizer; import org.openmetadata.catalog.security.SecurityUtil; +import org.openmetadata.catalog.tests.ColumnTest; +import org.openmetadata.catalog.tests.TableTest; import org.openmetadata.catalog.type.DataModel; import org.openmetadata.catalog.type.EntityHistory; import org.openmetadata.catalog.type.Include; @@ -106,7 +108,7 @@ public class TableResource { static final String FIELDS = "columns,tableConstraints,usageSummary,owner," - + "tags,followers,joins,sampleData,viewDefinition,tableProfile,location,tableQueries,dataModel"; + + "tags,followers,joins,sampleData,viewDefinition,tableProfile,location,tableQueries,dataModel,tests"; public static final List FIELD_LIST = Arrays.asList(FIELDS.replace(" ", "").split(",")); @GET @@ -502,6 +504,34 @@ public class TableResource { return addHref(uriInfo, table); } + @PUT + @Path("/{id}/tableTest") + @Operation(summary = "Add table test cases", tags = "tables", description = "Add test cases to the table.") + public Table addTableTest( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id, + TableTest tableTest) + throws IOException, ParseException { + SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + Table table = dao.addTableTest(UUID.fromString(id), tableTest); + return addHref(uriInfo, table); + } + + @PUT + @Path("/{id}/columnTest") + @Operation(summary = "Add table test cases", tags = "tables", description = "Add test cases to the table.") + public Table addColumnTest( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id, + ColumnTest columnTest) + throws IOException, ParseException { + SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + Table table = dao.addColumnTest(UUID.fromString(id), columnTest); + return addHref(uriInfo, table); + } + @DELETE @Path("/{id}/followers/{userId}") @Operation( diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json index f3ebccc9b19..8ea3925a110 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json @@ -179,6 +179,14 @@ "$ref": "#/definitions/column" }, "default": null + }, + "columnTests": { + "description": "List of column test cases that ran against a table column.", + "type": "array", + "items": { + "$ref": "../../tests/columnTest.json" + }, + "default": null } }, "required": ["name", "dataType"], @@ -588,6 +596,14 @@ }, "default": null }, + "tableTests": { + "description": "List of test cases that ran against a table.", + "type": "array", + "items": { + "$ref": "../../tests/tableTest.json" + }, + "default": null + }, "dataModel": { "description": "This captures information about how the table is modeled. Currently only DBT model is supported.", "$ref": "#/definitions/dataModel" diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/basic.json b/catalog-rest-service/src/main/resources/json/schema/tests/basic.json new file mode 100644 index 00000000000..52f470b5ff9 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/basic.json @@ -0,0 +1,61 @@ +{ + "$id": "https://open-metadata.org/schema/tests/basic.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Basic", + "description": "This schema defines basic types that are used by other test schemas.", + "definitions": { + "testCaseResult": { + "description": "Schema to capture test case result.", + "javaType": "org.openmetadata.catalog.tests.type.TestCaseResult", + "type": "object", + "properties": { + "executionTime": { + "description": "Data one which profile is taken.", + "$ref": "../type/basic.json#/definitions/timestamp" + }, + "status": { + "description": "Status of Test Case run.", + "javaType": "org.openmetadata.catalog.tests.type.TestCaseStatus", + "type": "string", + "enum": ["Success", "Failed", "Aborted"], + "javaEnums": [ + { + "name": "Success" + }, + { + "name": "Failed" + }, + { + "name": "Aborted" + } + ] + }, + "result": { + "description": "Details of test case results.", + "type": "string" + }, + "sampleData": { + "description": "sample data to capture rows/columns that didn't match the expressed testcase.", + "type": "string" + } + } + }, + "testCaseExecutionFrequency": { + "description": "How often the test case should run.", + "javaType": "org.openmetadata.catalog.tests.type.TestCaseExecutionFrequency", + "type": "string", + "enum": ["Hourly", "Daily", "Weekly"], + "javaEnums": [ + { + "name": "Hourly" + }, + { + "name": "Daily" + }, + { + "name": "Weekly" + } + ] + } + } +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesLengthsToBeBetween.json b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesLengthsToBeBetween.json new file mode 100644 index 00000000000..fb262ec5369 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesLengthsToBeBetween.json @@ -0,0 +1,27 @@ +{ + "$id": "https://open-metadata.org/schema/tests/column/columnValueLengthsToBeBetween.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "columnValueLengthsToBeBetween", + "description": "This schema defines the test ColumnValueLengthsToBeBetween. Test the value lengths in a column to be between minimum and maximum value. ", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.column.ColumnValueLengthsToBeBetween", + "properties": { + "minValue": { + "description": "The {minValue} for the column length. If minValue is not included, maxValue is treated as upperBound and there will be no minimum number of rows", + "type": "integer" + }, + "maxValue": { + "description": "The {maxValue} for the column length. if maxValue is not included, minValue is treated as lowerBound and there will eb no maximum number of rows", + "type": "integer" + } + }, + "anyOf": [ + { + "required": ["minValue"] + }, + { + "required": ["maxValue"] + } + ], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesMissingCountToBeEqual.json b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesMissingCountToBeEqual.json new file mode 100644 index 00000000000..c85dd4d7811 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesMissingCountToBeEqual.json @@ -0,0 +1,22 @@ +{ + "$id": "https://open-metadata.org/schema/tests/column/columnValuesMissingCountToBeEqual.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "columnValuesMissingCount", + "description": "This schema defines the test ColumnValuesMissingCount. Test the column values missing count to be equal to given number. ", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.column.ColumnValuesMissingCountToBeEqual", + "properties": { + "missingCountValue": { + "description": "No.of missing values to be equal to.", + "type": "integer" + }, + "missingValueMatch": { + "description": "By default match all null and empty values to be missing. This field allows us to configure additional strings such as N/A, NULL as missing strings as well.", + "items": { + "type": "string" + } + } + }, + "required": ["missingValue"], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeBetween.json b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeBetween.json new file mode 100644 index 00000000000..c75e520baac --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeBetween.json @@ -0,0 +1,20 @@ +{ + "$id": "https://open-metadata.org/schema/tests/column/columnValuesToBeBetween.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "columnValuesToBeBetween", + "description": "This schema defines the test ColumnValuesToBeBetween. Test the values in a column to be between minimum and maximum value. ", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.column.ColumnValuesToBeBetween", + "properties": { + "minValue": { + "description": "The {minValue} value for the column entry. If minValue is not included, maxValue is treated as upperBound and there will be no minimum number of rows", + "type": "integer" + }, + "maxValue": { + "description": "The {maxValue} value for the column entry. if maxValue is not included, minValue is treated as lowerBound and there will eb no maximum number of rows", + "type": "integer" + } + }, + "anyOf": [{ "required": ["minValue"] }, { "required": ["maxValue"] }], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeNotInSet.json b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeNotInSet.json new file mode 100644 index 00000000000..ac9e548d963 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeNotInSet.json @@ -0,0 +1,18 @@ +{ + "$id": "https://open-metadata.org/schema/tests/column/columnValuesToBeNotInSet.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "columnValuesToBeNotInSet", + "description": "This schema defines the test ColumnValuesToBeNotInSet. Test the column values to not be in the set. ", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.column.ColumnValuesToBeNotInSet", + "properties": { + "values": { + "description": "An Array of values.", + "items": { + "type": "object" + } + } + }, + "required": ["values"], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeNotNull.json b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeNotNull.json new file mode 100644 index 00000000000..32d1fe48951 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeNotNull.json @@ -0,0 +1,10 @@ +{ + "$id": "https://open-metadata.org/schema/tests/column/columnValuesToBeNotNull.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "columnValuesToBeNotNull", + "description": "This schema defines the test ColumnValuesToBeNotNull. Test the number of values in a column are null. Values must be explicitly null. Empty strings don't count as null. ", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.column.ColumnValuesToBeNotNull", + "properties": {}, + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeUnique.json b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeUnique.json new file mode 100644 index 00000000000..94bb03e6bb7 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeUnique.json @@ -0,0 +1,10 @@ +{ + "$id": "https://open-metadata.org/schema/tests/column/columnValuesToBeUnique.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "columnValuesToBeUnique", + "description": "This schema defines the test ColumnValuesToBeUnique. Test the values in a column to be unique. ", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.column.ColumnValuesToBeUnique", + "properties": {}, + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToMatchRegex.json b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToMatchRegex.json new file mode 100644 index 00000000000..79505a8f294 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToMatchRegex.json @@ -0,0 +1,16 @@ +{ + "$id": "https://open-metadata.org/schema/tests/column/columnValuesToMatchRegex.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "columnValuesToBeUnique", + "description": "This schema defines the test ColumnValuesToMatchRegex. Test the values in a column to match a given regular expression. ", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.column.ColumnValuesToMatchRegex", + "properties": { + "regex": { + "description": "The regular expression the column entries should match.", + "type": "string" + } + }, + "required": ["regex"], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json b/catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json new file mode 100644 index 00000000000..021dd0a63e6 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json @@ -0,0 +1,106 @@ +{ + "$id": "https://open-metadata.org/schema/tests/columnTest.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ColumnTest", + "description": "ColumnTest is a test definition to capture data quality tests against tables and columns.", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.ColumnTest", + "definitions": { + "columnTestCase": { + "description": "Column Test Case.", + "type": "object", + "properties": { + "config": { + "oneOf": [ + { + "$ref": "./column/columnValuesToBeUnique.json" + }, + { + "$ref": "./column/columnValuesToBeNotNull.json" + }, + { + "$ref": "./column/columnValuesToMatchRegex.json" + }, + { + "$ref": "./column/columnValuesToBeNotInSet.json" + }, + { + "$ref": "column/columnValuesToBeBetween.json" + }, + { + "$ref": "./column/columnValuesMissingCountToBeEqual.json" + }, + { + "$ref": "./column/columnValuesLengthsToBeBetween.json" + } + ] + }, + "testType": { + "enum": [ + "columnValuesToBeUnique", + "columnValuesToBeNotNull", + "columnValuesToMatchRegex", + "columnValuesToBeNotInSet", + "columnValuesToBeBetween", + "columnValuesMissingCountToBeEqual", + "columnValueLengthsToBeBetween" + ] + } + }, + "additionalProperties": false + } + }, + "properties": { + "id": { + "description": "Unique identifier of this table instance.", + "$ref": "../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name that identifies this test case. Name passed by client will be overridden by auto generating based on table/column name and test name", + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "description": { + "description": "Description of the testcase.", + "type": "string" + }, + "columnName": { + "description": "Name of the column in a table.", + "type": "string" + }, + "testCase": { + "$ref": "#/definitions/columnTestCase" + }, + "executionFrequency": { + "$ref": "./basic.json#/definitions/testCaseExecutionFrequency" + }, + "results": { + "description": "List of results of the test case.", + "type": "array", + "items": { + "$ref": "./basic.json#/definitions/testCaseResult" + } + }, + "owner": { + "description": "Owner of this Pipeline.", + "$ref": "../type/entityReference.json", + "default": null + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "deleted": { + "description": "When `true` indicates the entity has been soft deleted.", + "type": "boolean", + "default": false + } + }, + "required": ["name", "column", "testCase"], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json new file mode 100644 index 00000000000..3f2e20154cc --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json @@ -0,0 +1,16 @@ +{ + "$id": "https://open-metadata.org/schema/tests/tableColumnCountToEqual.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TableRowCountToEqual", + "description": "This scheam defines the test TableColumnCountToEqual. Test the number of columns equal to a value.", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.table.TableColumnCountToEqual", + "properties": { + "value": { + "description": "Expected number of columns to equal to a {value}", + "type": "integer" + } + }, + "required": ["value"], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/table/tableRowCountToBeBetween.json b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableRowCountToBeBetween.json new file mode 100644 index 00000000000..a5fa0d12dc2 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableRowCountToBeBetween.json @@ -0,0 +1,20 @@ +{ + "$id": "https://open-metadata.org/schema/tests/tableRowCountToBeBetween.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TableRowCountToEqual", + "description": "This scheam defines the test TableRowCountToBeBetween. Test the number of rows to between to two values.", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.table.TableRowCountToBeBetween", + "properties": { + "minValue": { + "description": "Expected number of rows should be greater than or equal to {minValue}. If minValue is not included, maxValue is treated as upperBound and there will be no minimum number of rows", + "type": "integer" + }, + "maxValue": { + "description": "Expected number of rows should be lower than or equal to {maxValue}. if maxValue is not included, minValue is treated as lowerBound and there will eb no maximum number of rows", + "type": "integer" + } + }, + "anyOf": [{ "required": ["minValue"] }, { "required": ["maxValue"] }], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/table/tableRowCountToEqual.json b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableRowCountToEqual.json new file mode 100644 index 00000000000..c6168557563 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableRowCountToEqual.json @@ -0,0 +1,16 @@ +{ + "$id": "https://open-metadata.org/schema/tests/table/tableRowCountToEqual.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TableRowCountToEqual", + "description": "This schema defines the test TableRowCountToEqual. Test the number of rows equal to a value.", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.table.TableRowCountToEqual", + "properties": { + "value": { + "description": "Expected number of rows {value}", + "type": "integer" + } + }, + "required": ["value"], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/tableTest.json b/catalog-rest-service/src/main/resources/json/schema/tests/tableTest.json new file mode 100644 index 00000000000..e434f7df0ed --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/tableTest.json @@ -0,0 +1,90 @@ +{ + "$id": "https://open-metadata.org/schema/tests/tableTest.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TableTest", + "description": "TableTest is a test definition to capture data quality tests against tables and columns.", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.TableTest", + "definitions": { + "tableTestCase": { + "description": "Table Test Case.", + "type": "object", + "properties": { + "config": { + "oneOf": [ + { + "$ref": "./table/tableRowCountToEqual.json" + }, + { + "$ref": "./table/tableRowCountToBeBetween.json" + }, + { + "$ref": "./table/tableColumnCountToEqual.json" + } + ] + }, + "testType": { + "enum": [ + "tableRowCountToEqual", + "tableRowCountToBeBetween", + "tableColumnCountToEqual" + ] + } + }, + "additionalProperties": false + } + }, + "properties": { + "id": { + "description": "Unique identifier of this table instance.", + "$ref": "../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name that identifies this test case.", + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "description": { + "description": "Description of the testcase.", + "type": "string" + }, + "tableName": { + "description": "Table Name for which this test applies.", + "type": "string" + }, + "tableTestCase": { + "$ref": "#/definitions/tableTestCase" + }, + "executionFrequency": { + "$ref": "./basic.json#/definitions/testCaseExecutionFrequency" + }, + "results": { + "description": "List of results of the test case.", + "type": "array", + "items": { + "$ref": "./basic.json#/definitions/testCaseResult" + } + }, + "owner": { + "description": "Owner of this Pipeline.", + "$ref": "../type/entityReference.json", + "default": null + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "deleted": { + "description": "When `true` indicates the entity has been soft deleted.", + "type": "boolean", + "default": false + } + }, + "required": ["name", "testCase", "tableName"], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java index d6bd8eb4c22..ea1b455f781 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java @@ -85,6 +85,21 @@ import org.openmetadata.catalog.resources.databases.TableResource.TableList; import org.openmetadata.catalog.resources.locations.LocationResourceTest; import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest; import org.openmetadata.catalog.resources.tags.TagResourceTest; +import org.openmetadata.catalog.tests.ColumnTest; +import org.openmetadata.catalog.tests.ColumnTestCase; +import org.openmetadata.catalog.tests.TableTest; +import org.openmetadata.catalog.tests.TableTestCase; +import org.openmetadata.catalog.tests.column.ColumnValueLengthsToBeBetween; +import org.openmetadata.catalog.tests.column.ColumnValuesMissingCountToBeEqual; +import org.openmetadata.catalog.tests.column.ColumnValuesToBeNotNull; +import org.openmetadata.catalog.tests.column.ColumnValuesToBeUnique; +import org.openmetadata.catalog.tests.column.ColumnValuesToMatchRegex; +import org.openmetadata.catalog.tests.table.TableColumnCountToEqual; +import org.openmetadata.catalog.tests.table.TableRowCountToBeBetween; +import org.openmetadata.catalog.tests.table.TableRowCountToEqual; +import org.openmetadata.catalog.tests.type.TestCaseExecutionFrequency; +import org.openmetadata.catalog.tests.type.TestCaseResult; +import org.openmetadata.catalog.tests.type.TestCaseStatus; import org.openmetadata.catalog.type.ChangeDescription; import org.openmetadata.catalog.type.Column; import org.openmetadata.catalog.type.ColumnConstraint; @@ -1019,6 +1034,119 @@ public class TableResourceTest extends EntityResourceTest { assertEquals(expected.getGeneratedAt(), actual.getGeneratedAt()); } + @Test + void put_tableColumnTests_200(TestInfo test) throws IOException { + Table table = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS); + TableRowCountToEqual tableRowCountToEqual = new TableRowCountToEqual().withValue(100); + TableTestCase tableTestCase = + new TableTestCase() + .withTestType(TableTestCase.TestType.TABLE_ROW_COUNT_TO_EQUAL) + .withConfig(tableRowCountToEqual); + TableTest tableTest = + new TableTest() + .withName("test1") + .withTableTestCase(tableTestCase) + .withExecutionFrequency(TestCaseExecutionFrequency.Hourly); + Table putResponse = putTableTest(table.getId(), tableTest, ADMIN_AUTH_HEADERS); + verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(tableTest)); + + table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS); + verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest)); + + // Add result to tableTest + TestCaseResult testCaseResult1 = + new TestCaseResult() + .withResult("Rows equal to 100") + .withStatus(TestCaseStatus.Success) + .withSampleData("Rows == 100") + .withExecutionTime(100L); + tableTest.setResults(List.of(testCaseResult1)); + tableTest.setId(table.getTableTests().get(0).getId()); + putResponse = putTableTest(table.getId(), tableTest, ADMIN_AUTH_HEADERS); + verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(tableTest)); + + TestCaseResult testCaseResult2 = + new TestCaseResult() + .withResult("Rows equal to 100") + .withStatus(TestCaseStatus.Success) + .withSampleData("Rows == 100") + .withExecutionTime(100L); + tableTest.setResults(List.of(testCaseResult2)); + tableTest.setId(table.getTableTests().get(0).getId()); + + table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS); + verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest)); + TableRowCountToBeBetween tableRowCountToBeBetween = + new TableRowCountToBeBetween().withMinValue(100).withMaxValue(1000); + TableTestCase tableTestCase1 = + new TableTestCase() + .withTestType(TableTestCase.TestType.TABLE_ROW_COUNT_TO_BE_BETWEEN) + .withConfig(tableRowCountToBeBetween); + TableTest tableTest1 = new TableTest().withName("column_value_to_be_unique").withTableTestCase(tableTestCase1); + putResponse = putTableTest(table.getId(), tableTest1, ADMIN_AUTH_HEADERS); + verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(tableTest, tableTest1)); + table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS); + verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest, tableTest1)); + + Column c1 = table.getColumns().get(0); + ColumnValueLengthsToBeBetween columnValueLengthsToBeBetween = + new ColumnValueLengthsToBeBetween().withMaxValue(100).withMinValue(10); + ColumnTestCase columnTestCase = + new ColumnTestCase() + .withTestType(ColumnTestCase.TestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN) + .withConfig(columnValueLengthsToBeBetween); + ColumnTest columnTest = + new ColumnTest() + .withColumnName(c1.getName()) + .withName("test") + .withTestCase(columnTestCase) + .withExecutionFrequency(TestCaseExecutionFrequency.Hourly); + putResponse = putColumnTest(table.getId(), columnTest, ADMIN_AUTH_HEADERS); + verifyColumnTest(putResponse, c1, List.of(columnTest)); + + table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS); + verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest, tableTest1)); + verifyColumnTest(table, c1, List.of(columnTest)); + + // Add result to columnTest + TestCaseResult colTestCaseResult = + new TestCaseResult() + .withResult("min is > 100 and max < 1000") + .withStatus(TestCaseStatus.Success) + .withSampleData("minValue is 100 and maxValue is 1000") + .withExecutionTime(100L); + columnTest.setResults(List.of(colTestCaseResult)); + putResponse = putColumnTest(table.getId(), columnTest, ADMIN_AUTH_HEADERS); + verifyColumnTest(putResponse, c1, List.of(columnTest)); + + ColumnValuesMissingCountToBeEqual columnValuesMissingCountToBeEqual = + new ColumnValuesMissingCountToBeEqual().withMissingCountValue(10); + ColumnTestCase columnTestCase1 = + new ColumnTestCase() + .withTestType(ColumnTestCase.TestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL) + .withConfig(columnValuesMissingCountToBeEqual); + ColumnTest columnTest1 = + new ColumnTest() + .withColumnName(c1.getName()) + .withName("test") + .withTestCase(columnTestCase1) + .withExecutionFrequency(TestCaseExecutionFrequency.Hourly); + putResponse = putColumnTest(table.getId(), columnTest1, ADMIN_AUTH_HEADERS); + verifyColumnTest(putResponse, c1, List.of(columnTest, columnTest1)); + + // Add result to columnTest + TestCaseResult colTestCaseResult1 = + new TestCaseResult() + .withResult("min is > 100 and max < 1000") + .withStatus(TestCaseStatus.Success) + .withSampleData("minValue is 100 and maxValue is 1000") + .withExecutionTime(100L); + columnTest.setResults(List.of(colTestCaseResult1)); + putResponse = putColumnTest(table.getId(), columnTest, ADMIN_AUTH_HEADERS); + columnTest.setResults(List.of(colTestCaseResult, colTestCaseResult1)); + verifyColumnTest(putResponse, c1, List.of(columnTest, columnTest1)); + } + @Test void get_deletedTableWithDeleteLocation(TestInfo test) throws IOException { CreateTable create = createRequest(getEntityName(test), "description", "displayName", USER_OWNER1); @@ -1500,6 +1628,18 @@ public class TableResourceTest extends EntityResourceTest { return TestUtils.put(target, dataModel, Table.class, OK, authHeaders); } + public static Table putTableTest(UUID tableId, TableTest data, Map authHeaders) + throws HttpResponseException { + WebTarget target = CatalogApplicationTest.getResource("tables/" + tableId + "/tableTest"); + return TestUtils.put(target, data, Table.class, OK, authHeaders); + } + + public static Table putColumnTest(UUID tableId, ColumnTest data, Map authHeaders) + throws HttpResponseException { + WebTarget target = CatalogApplicationTest.getResource("tables/" + tableId + "/columnTest"); + return TestUtils.put(target, data, Table.class, OK, authHeaders); + } + private static int getTagUsageCount(String tagFQN, Map authHeaders) throws HttpResponseException { return TagResourceTest.getTag(tagFQN, "usageCount", authHeaders).getUsageCount(); } @@ -1522,6 +1662,118 @@ public class TableResourceTest extends EntityResourceTest { } } + private void verifyTableTest(String tableName, List actualTests, List expectedTests) + throws IOException { + assertEquals(actualTests.size(), expectedTests.size()); + Map tableTestMap = new HashMap<>(); + for (TableTest test : actualTests) { + tableTestMap.put(test.getName(), test); + } + for (TableTest test : expectedTests) { + // passed in test name will be overridden in backend + String expectedTestName = tableName + "." + test.getTableTestCase().getTestType().toString(); + TableTest storedTest = tableTestMap.get(expectedTestName); + assertNotNull(storedTest); + assertEquals(expectedTestName, storedTest.getName()); + assertEquals(test.getDescription(), storedTest.getDescription()); + assertEquals(test.getExecutionFrequency(), storedTest.getExecutionFrequency()); + assertEquals(test.getOwner(), storedTest.getOwner()); + verifyTableTestCase(test.getTableTestCase(), storedTest.getTableTestCase()); + verifyTestCaseResults(test.getResults(), storedTest.getResults()); + } + } + + private void verifyTableTestCase(TableTestCase expected, TableTestCase actual) { + assertEquals(expected.getTestType(), actual.getTestType()); + if (expected.getTestType() == TableTestCase.TestType.TABLE_COLUMN_COUNT_TO_EQUAL) { + TableColumnCountToEqual expectedTest = (TableColumnCountToEqual) expected.getConfig(); + TableColumnCountToEqual actualTest = JsonUtils.convertValue(actual.getConfig(), TableColumnCountToEqual.class); + assertEquals(expectedTest.getValue(), actualTest.getValue()); + } else if (expected.getTestType() == TableTestCase.TestType.TABLE_ROW_COUNT_TO_BE_BETWEEN) { + TableRowCountToBeBetween expectedTest = (TableRowCountToBeBetween) expected.getConfig(); + TableRowCountToBeBetween actualTest = JsonUtils.convertValue(actual.getConfig(), TableRowCountToBeBetween.class); + assertEquals(expectedTest.getMaxValue(), actualTest.getMaxValue()); + assertEquals(expectedTest.getMinValue(), actualTest.getMinValue()); + } else if (expected.getTestType() == TableTestCase.TestType.TABLE_ROW_COUNT_TO_EQUAL) { + TableRowCountToEqual expectedTest = (TableRowCountToEqual) expected.getConfig(); + TableRowCountToEqual actualTest = JsonUtils.convertValue(actual.getConfig(), TableRowCountToEqual.class); + assertEquals(expectedTest.getValue(), actualTest.getValue()); + } + } + + private void verifyColumnTest(Table table, Column column, List expectedTests) throws IOException { + List actualTests = new ArrayList<>(); + for (Column c : table.getColumns()) { + if (c.getName().equals(column.getName())) { + actualTests = c.getColumnTests(); + } + } + assertEquals(actualTests.size(), expectedTests.size()); + + Map columnTestMap = new HashMap<>(); + for (ColumnTest test : actualTests) { + columnTestMap.put(test.getName(), test); + } + + for (ColumnTest test : expectedTests) { + // passed in test name will be overridden in backend + String expectedTestName = column.getName() + "." + test.getTestCase().getTestType().toString(); + ColumnTest storedTest = columnTestMap.get(expectedTestName); + assertNotNull(storedTest); + assertEquals(expectedTestName, storedTest.getName()); + assertEquals(test.getDescription(), storedTest.getDescription()); + assertEquals(test.getExecutionFrequency(), storedTest.getExecutionFrequency()); + assertEquals(test.getOwner(), storedTest.getOwner()); + verifyColumnTestCase(test.getTestCase(), storedTest.getTestCase()); + verifyTestCaseResults(test.getResults(), storedTest.getResults()); + } + } + + private void verifyColumnTestCase(ColumnTestCase expected, ColumnTestCase actual) { + assertEquals(expected.getTestType(), actual.getTestType()); + if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_TO_BE_UNIQUE) { + ColumnValuesToBeUnique expectedTest = (ColumnValuesToBeUnique) expected.getConfig(); + ColumnValuesToBeUnique actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesToBeUnique.class); + assertEquals(expectedTest, actualTest); + } else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_TO_BE_NOT_NULL) { + ColumnValuesToBeNotNull expectedTest = (ColumnValuesToBeNotNull) expected.getConfig(); + ColumnValuesToBeNotNull actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesToBeNotNull.class); + assertEquals(expectedTest, actualTest); + } else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_TO_MATCH_REGEX) { + ColumnValuesToMatchRegex expectedTest = (ColumnValuesToMatchRegex) expected.getConfig(); + ColumnValuesToMatchRegex actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesToMatchRegex.class); + assertEquals(expectedTest.getRegex(), actualTest.getRegex()); + } else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN) { + ColumnValueLengthsToBeBetween expectedTest = (ColumnValueLengthsToBeBetween) expected.getConfig(); + ColumnValueLengthsToBeBetween actualTest = + JsonUtils.convertValue(actual.getConfig(), ColumnValueLengthsToBeBetween.class); + assertEquals(expectedTest.getMaxValue(), actualTest.getMaxValue()); + assertEquals(expectedTest.getMinValue(), actualTest.getMinValue()); + } else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL) { + ColumnValuesMissingCountToBeEqual expectedTest = (ColumnValuesMissingCountToBeEqual) expected.getConfig(); + ColumnValuesMissingCountToBeEqual actualTest = + JsonUtils.convertValue(actual.getConfig(), ColumnValuesMissingCountToBeEqual.class); + assertEquals(expectedTest.getMissingCountValue(), actualTest.getMissingCountValue()); + assertEquals(expectedTest.getMissingValueMatch(), actualTest.getMissingValueMatch()); + } + } + + private void verifyTestCaseResults(List expected, List actual) throws IOException { + assertEquals(expected.size(), actual.size()); + Map actualResultMap = new HashMap<>(); + for (Object a : actual) { + TestCaseResult result = JsonUtils.convertValue(a, TestCaseResult.class); + actualResultMap.put(result.getExecutionTime(), result); + } + for (Object e : expected) { + TestCaseResult result = JsonUtils.convertValue(e, TestCaseResult.class); + TestCaseResult actualResult = actualResultMap.get(result.getExecutionTime()); + assertNotNull(actualResult); + assertEquals(result.getResult(), actualResult.getResult()); + assertEquals(result.getSampleData(), actualResult.getSampleData()); + } + } + @Override public CreateTable createRequest(String name, String description, String displayName, EntityReference owner) { TableConstraint constraint =