Fix #2873: DataQuality test specifications and APIs (#2906)

* Fix #2873: DataQuality test specifications and APIs

catalog-rest-service/src/main/java/org/openmetadata/catalog/events/ChangeEventHandler.java

@@ -182,7 +182,9 @@ public class ChangeEventHandler implements EventHandler {
     }
 
     var entityInterface = Entity.getEntityInterface(entity);
-
+    if (entityInterface.getChangeDescription() == null) {
+      return null;
+    }
     List<FieldChange> fieldsUpdated = entityInterface.getChangeDescription().getFieldsUpdated();
     List<Thread> threads = new ArrayList<>(getThreads(fieldsUpdated, entity, CHANGE_TYPE.UPDATE));
 

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java

@@ -49,6 +49,9 @@ import org.openmetadata.catalog.entity.data.Table;
 import org.openmetadata.catalog.entity.services.DatabaseService;
 import org.openmetadata.catalog.exception.CatalogExceptionMessage;
 import org.openmetadata.catalog.resources.databases.TableResource;
+import org.openmetadata.catalog.tests.ColumnTest;
+import org.openmetadata.catalog.tests.TableTest;
+import org.openmetadata.catalog.tests.type.TestCaseResult;
 import org.openmetadata.catalog.type.ChangeDescription;
 import org.openmetadata.catalog.type.Column;
 import org.openmetadata.catalog.type.ColumnJoin;
@@ -110,6 +113,8 @@ public class TableRepository extends EntityRepository<Table> {
     table.setTableProfile(fields.contains("tableProfile") ? getTableProfile(table) : null);
     table.setLocation(fields.contains("location") ? getLocation(table) : null);
     table.setTableQueries(fields.contains("tableQueries") ? getQueries(table) : null);
+    table.setTableTests(fields.contains("tests") ? getTableTests(table) : null);
+    getColumnTests(fields.contains("tests"), table);
     return table;
   }
 
@@ -245,6 +250,90 @@ public class TableRepository extends EntityRepository<Table> {
     return table.withTableQueries(getQueries(table));
   }
 
+  @Transaction
+  public Table addTableTest(UUID tableId, TableTest tableTest) throws IOException, ParseException {
+    // Validate the request content
+    Table table = daoCollection.tableDAO().findEntityById(tableId);
+    // if ID is not passed we treat it as a new test case being added
+    List<TableTest> storedTableTests = getTableTests(table);
+    // we will override any test case name passed by user/client with tableName + testType
+    // our assumption is there is only one instance of a test type as of now.
+    tableTest.setName(table.getName() + "." + tableTest.getTableTestCase().getTestType().toString());
+    Map<String, TableTest> storedMapTableTests = new HashMap<>();
+    if (storedTableTests != null) {
+      for (TableTest t : storedTableTests) {
+        storedMapTableTests.put(t.getName(), t);
+      }
+    }
+    // new test add UUID
+    if (!storedMapTableTests.containsKey(tableTest.getName())) {
+      tableTest.setId(UUID.randomUUID());
+    }
+
+    // process test result
+    if (storedMapTableTests.containsKey(tableTest.getName())
+        && tableTest.getResults() != null
+        && !tableTest.getResults().isEmpty()) {
+      TableTest prevTableTest = storedMapTableTests.get(tableTest.getName());
+      List<TestCaseResult> prevTestCaseResults = prevTableTest.getResults();
+      List<TestCaseResult> newTestCaseResults = tableTest.getResults();
+      newTestCaseResults.addAll(prevTestCaseResults);
+      tableTest.setResults(newTestCaseResults);
+    }
+
+    storedMapTableTests.put(tableTest.getName(), tableTest);
+    List<TableTest> updatedTests = new ArrayList<>(storedMapTableTests.values());
+    daoCollection
+        .entityExtensionDAO()
+        .insert(tableId.toString(), "table.tableTests", "tableTest", JsonUtils.pojoToJson(updatedTests));
+    setFields(table, Fields.EMPTY_FIELDS);
+    return table.withTableTests(getTableTests(table));
+  }
+
+  @Transaction
+  public Table addColumnTest(UUID tableId, ColumnTest columnTest) throws IOException, ParseException {
+    // Validate the request content
+    Table table = daoCollection.tableDAO().findEntityById(tableId);
+    String columnName = columnTest.getColumnName();
+    validateColumn(table, columnName);
+    // we will override any test case name passed by user/client with columnName + testType
+    // our assumption is there is only one instance of a test type as of now.
+    columnTest.setName(columnName + "." + columnTest.getTestCase().getTestType().toString());
+    List<ColumnTest> storedColumnTests = getColumnTests(table, columnName);
+    Map<String, ColumnTest> storedMapColumnTests = new HashMap<>();
+    if (storedColumnTests != null) {
+      for (ColumnTest ct : storedColumnTests) {
+        storedMapColumnTests.put(ct.getName(), ct);
+      }
+    }
+
+    // new test, generate UUID
+    if (!storedMapColumnTests.containsKey(columnTest.getName())) {
+      columnTest.setId(UUID.randomUUID());
+    }
+
+    // process test result
+    if (storedMapColumnTests.containsKey(columnTest.getName())
+        && columnTest.getResults() != null
+        && !columnTest.getResults().isEmpty()) {
+      ColumnTest prevColumnTest = storedMapColumnTests.get(columnTest.getName());
+      List<TestCaseResult> prevTestCaseResults = prevColumnTest.getResults();
+      List<TestCaseResult> newTestCaseResults = columnTest.getResults();
+      newTestCaseResults.addAll(prevTestCaseResults);
+      columnTest.setResults(newTestCaseResults);
+    }
+
+    storedMapColumnTests.put(columnTest.getName(), columnTest);
+    List<ColumnTest> updatedTests = new ArrayList<>(storedMapColumnTests.values());
+    String extension = "table.column." + columnName + ".tests";
+    daoCollection
+        .entityExtensionDAO()
+        .insert(table.getId().toString(), extension, "columnTest", JsonUtils.pojoToJson(updatedTests));
+    setFields(table, Fields.EMPTY_FIELDS);
+    getColumnTests(true, table);
+    return table;
+  }
+
   @Transaction
   public Table addDataModel(UUID tableId, DataModel dataModel) throws IOException, ParseException {
     Table table = daoCollection.tableDAO().findEntityById(tableId);
@@ -642,6 +731,24 @@ public class TableRepository extends EntityRepository<Table> {
     return tableQueries;
   }
 
+  private List<TableTest> getTableTests(Table table) throws IOException {
+    return JsonUtils.readObjects(
+        daoCollection.entityExtensionDAO().getExtension(table.getId().toString(), "table.tableTests"), TableTest.class);
+  }
+
+  private List<ColumnTest> getColumnTests(Table table, String columnName) throws IOException {
+    String extension = "table.column." + columnName + ".tests";
+    return JsonUtils.readObjects(
+        daoCollection.entityExtensionDAO().getExtension(table.getId().toString(), extension), ColumnTest.class);
+  }
+
+  private void getColumnTests(boolean setTests, Table table) throws IOException {
+    List<Column> columns = table.getColumns();
+    for (Column c : Optional.ofNullable(columns).orElse(Collections.emptyList())) {
+      c.setColumnTests(setTests ? getColumnTests(table, c.getName()) : null);
+    }
+  }
+
   public static class TableEntityInterface implements EntityInterface<Table> {
     private final Table entity;
 

catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java

@@ -57,6 +57,8 @@ import org.openmetadata.catalog.jdbi3.TableRepository;
 import org.openmetadata.catalog.resources.Collection;
 import org.openmetadata.catalog.security.Authorizer;
 import org.openmetadata.catalog.security.SecurityUtil;
+import org.openmetadata.catalog.tests.ColumnTest;
+import org.openmetadata.catalog.tests.TableTest;
 import org.openmetadata.catalog.type.DataModel;
 import org.openmetadata.catalog.type.EntityHistory;
 import org.openmetadata.catalog.type.Include;
@@ -106,7 +108,7 @@ public class TableResource {
 
   static final String FIELDS =
       "columns,tableConstraints,usageSummary,owner,"
-          + "tags,followers,joins,sampleData,viewDefinition,tableProfile,location,tableQueries,dataModel";
+          + "tags,followers,joins,sampleData,viewDefinition,tableProfile,location,tableQueries,dataModel,tests";
   public static final List<String> FIELD_LIST = Arrays.asList(FIELDS.replace(" ", "").split(","));
 
   @GET
@@ -502,6 +504,34 @@ public class TableResource {
     return addHref(uriInfo, table);
   }
 
+  @PUT
+  @Path("/{id}/tableTest")
+  @Operation(summary = "Add table test cases", tags = "tables", description = "Add test cases to the table.")
+  public Table addTableTest(
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext,
+      @Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id,
+      TableTest tableTest)
+      throws IOException, ParseException {
+    SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
+    Table table = dao.addTableTest(UUID.fromString(id), tableTest);
+    return addHref(uriInfo, table);
+  }
+
+  @PUT
+  @Path("/{id}/columnTest")
+  @Operation(summary = "Add table test cases", tags = "tables", description = "Add test cases to the table.")
+  public Table addColumnTest(
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext,
+      @Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id,
+      ColumnTest columnTest)
+      throws IOException, ParseException {
+    SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
+    Table table = dao.addColumnTest(UUID.fromString(id), columnTest);
+    return addHref(uriInfo, table);
+  }
+
   @DELETE
   @Path("/{id}/followers/{userId}")
   @Operation(

catalog-rest-service/src/main/resources/json/schema/entity/data/table.json

@@ -179,6 +179,14 @@
             "$ref": "#/definitions/column"
           },
           "default": null
+        },
+        "columnTests": {
+          "description": "List of column test cases that ran against a table column.",
+          "type": "array",
+          "items": {
+            "$ref": "../../tests/columnTest.json"
+          },
+          "default": null
         }
       },
       "required": ["name", "dataType"],
@@ -588,6 +596,14 @@
       },
       "default": null
     },
+    "tableTests": {
+      "description": "List of test cases that ran against a table.",
+      "type": "array",
+      "items": {
+        "$ref": "../../tests/tableTest.json"
+      },
+      "default": null
+    },
     "dataModel": {
       "description": "This captures information about how the table is modeled. Currently only DBT model is supported.",
       "$ref": "#/definitions/dataModel"

catalog-rest-service/src/main/resources/json/schema/tests/basic.json

@@ -0,0 +1,61 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/basic.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Basic",
+  "description": "This schema defines basic types that are used by other test schemas.",
+  "definitions": {
+    "testCaseResult": {
+      "description": "Schema to capture test case result.",
+      "javaType": "org.openmetadata.catalog.tests.type.TestCaseResult",
+      "type": "object",
+      "properties": {
+        "executionTime": {
+          "description": "Data one which profile is taken.",
+          "$ref": "../type/basic.json#/definitions/timestamp"
+        },
+        "status": {
+          "description": "Status of Test Case run.",
+          "javaType": "org.openmetadata.catalog.tests.type.TestCaseStatus",
+          "type": "string",
+          "enum": ["Success", "Failed", "Aborted"],
+          "javaEnums": [
+            {
+              "name": "Success"
+            },
+            {
+              "name": "Failed"
+            },
+            {
+              "name": "Aborted"
+            }
+          ]
+        },
+        "result": {
+          "description": "Details of test case results.",
+          "type": "string"
+        },
+        "sampleData": {
+          "description": "sample data to capture rows/columns that didn't match the expressed testcase.",
+          "type": "string"
+        }
+      }
+    },
+    "testCaseExecutionFrequency": {
+      "description": "How often the test case should run.",
+      "javaType": "org.openmetadata.catalog.tests.type.TestCaseExecutionFrequency",
+      "type": "string",
+      "enum": ["Hourly", "Daily", "Weekly"],
+      "javaEnums": [
+        {
+          "name": "Hourly"
+        },
+        {
+          "name": "Daily"
+        },
+        {
+          "name": "Weekly"
+        }
+      ]
+    }
+  }
+}

catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesLengthsToBeBetween.json

@@ -0,0 +1,27 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/column/columnValueLengthsToBeBetween.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "columnValueLengthsToBeBetween",
+  "description": "This schema defines the test ColumnValueLengthsToBeBetween. Test the value lengths in a column to be between minimum and maximum value. ",
+  "type": "object",
+  "javaType": "org.openmetadata.catalog.tests.column.ColumnValueLengthsToBeBetween",
+  "properties": {
+    "minValue": {
+      "description": "The {minValue} for the column length. If minValue is not included, maxValue is treated as upperBound and there will be no minimum number of rows",
+      "type": "integer"
+    },
+    "maxValue": {
+      "description": "The {maxValue} for the column length. if maxValue is not included, minValue is treated as lowerBound and there will eb no maximum number of rows",
+      "type": "integer"
+    }
+  },
+  "anyOf": [
+    {
+      "required": ["minValue"]
+    },
+    {
+      "required": ["maxValue"]
+    }
+  ],
+  "additionalProperties": false
+}

catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesMissingCountToBeEqual.json

@@ -0,0 +1,22 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/column/columnValuesMissingCountToBeEqual.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "columnValuesMissingCount",
+  "description": "This schema defines the test ColumnValuesMissingCount. Test the column values missing count to be equal to given number. ",
+  "type": "object",
+  "javaType": "org.openmetadata.catalog.tests.column.ColumnValuesMissingCountToBeEqual",
+  "properties": {
+    "missingCountValue": {
+      "description": "No.of missing values to be equal to.",
+      "type": "integer"
+    },
+    "missingValueMatch": {
+      "description": "By default match all null and empty values to be missing. This field allows us to configure additional strings such as N/A, NULL as missing strings as well.",
+      "items": {
+        "type": "string"
+      }
+    }
+  },
+  "required": ["missingValue"],
+  "additionalProperties": false
+}

catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeBetween.json

@@ -0,0 +1,20 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/column/columnValuesToBeBetween.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "columnValuesToBeBetween",
+  "description": "This schema defines the test ColumnValuesToBeBetween. Test the values in a column to be between minimum and maximum value. ",
+  "type": "object",
+  "javaType": "org.openmetadata.catalog.tests.column.ColumnValuesToBeBetween",
+  "properties": {
+    "minValue": {
+      "description": "The {minValue} value for the column entry. If minValue is not included, maxValue is treated as upperBound and there will be no minimum number of rows",
+      "type": "integer"
+    },
+    "maxValue": {
+      "description": "The {maxValue} value for the column entry. if maxValue is not included, minValue is treated as lowerBound and there will eb no maximum number of rows",
+      "type": "integer"
+    }
+  },
+  "anyOf": [{ "required": ["minValue"] }, { "required": ["maxValue"] }],
+  "additionalProperties": false
+}

catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeNotInSet.json

@@ -0,0 +1,18 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/column/columnValuesToBeNotInSet.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "columnValuesToBeNotInSet",
+  "description": "This schema defines the test ColumnValuesToBeNotInSet. Test the column values to not be in the set. ",
+  "type": "object",
+  "javaType": "org.openmetadata.catalog.tests.column.ColumnValuesToBeNotInSet",
+  "properties": {
+    "values": {
+      "description": "An Array of values.",
+      "items": {
+        "type": "object"
+      }
+    }
+  },
+  "required": ["values"],
+  "additionalProperties": false
+}

catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeNotNull.json

@@ -0,0 +1,10 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/column/columnValuesToBeNotNull.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "columnValuesToBeNotNull",
+  "description": "This schema defines the test ColumnValuesToBeNotNull. Test the number of values in a column are null. Values must be explicitly null. Empty strings don't count as null. ",
+  "type": "object",
+  "javaType": "org.openmetadata.catalog.tests.column.ColumnValuesToBeNotNull",
+  "properties": {},
+  "additionalProperties": false
+}

catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToBeUnique.json

@@ -0,0 +1,10 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/column/columnValuesToBeUnique.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "columnValuesToBeUnique",
+  "description": "This schema defines the test ColumnValuesToBeUnique. Test the values in a column to be unique. ",
+  "type": "object",
+  "javaType": "org.openmetadata.catalog.tests.column.ColumnValuesToBeUnique",
+  "properties": {},
+  "additionalProperties": false
+}

catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesToMatchRegex.json

@@ -0,0 +1,16 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/column/columnValuesToMatchRegex.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "columnValuesToBeUnique",
+  "description": "This schema defines the test ColumnValuesToMatchRegex. Test the values in a column to match a given regular expression. ",
+  "type": "object",
+  "javaType": "org.openmetadata.catalog.tests.column.ColumnValuesToMatchRegex",
+  "properties": {
+    "regex": {
+      "description": "The regular expression the column entries should match.",
+      "type": "string"
+    }
+  },
+  "required": ["regex"],
+  "additionalProperties": false
+}

catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json

@@ -0,0 +1,106 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/columnTest.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "ColumnTest",
+  "description": "ColumnTest is a test definition to capture data quality tests against tables and columns.",
+  "type": "object",
+  "javaType": "org.openmetadata.catalog.tests.ColumnTest",
+  "definitions": {
+    "columnTestCase": {
+      "description": "Column Test Case.",
+      "type": "object",
+      "properties": {
+        "config": {
+          "oneOf": [
+            {
+              "$ref": "./column/columnValuesToBeUnique.json"
+            },
+            {
+              "$ref": "./column/columnValuesToBeNotNull.json"
+            },
+            {
+              "$ref": "./column/columnValuesToMatchRegex.json"
+            },
+            {
+              "$ref": "./column/columnValuesToBeNotInSet.json"
+            },
+            {
+              "$ref": "column/columnValuesToBeBetween.json"
+            },
+            {
+              "$ref": "./column/columnValuesMissingCountToBeEqual.json"
+            },
+            {
+              "$ref": "./column/columnValuesLengthsToBeBetween.json"
+            }
+          ]
+        },
+        "testType": {
+          "enum": [
+            "columnValuesToBeUnique",
+            "columnValuesToBeNotNull",
+            "columnValuesToMatchRegex",
+            "columnValuesToBeNotInSet",
+            "columnValuesToBeBetween",
+            "columnValuesMissingCountToBeEqual",
+            "columnValueLengthsToBeBetween"
+          ]
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "properties": {
+    "id": {
+      "description": "Unique identifier of this table instance.",
+      "$ref": "../type/basic.json#/definitions/uuid"
+    },
+    "name": {
+      "description": "Name that identifies this test case. Name passed by client will be  overridden by  auto generating based on table/column name and test name",
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 128
+    },
+    "description": {
+      "description": "Description of the testcase.",
+      "type": "string"
+    },
+    "columnName": {
+      "description": "Name of the column in a table.",
+      "type": "string"
+    },
+    "testCase": {
+      "$ref": "#/definitions/columnTestCase"
+    },
+    "executionFrequency": {
+      "$ref": "./basic.json#/definitions/testCaseExecutionFrequency"
+    },
+    "results": {
+      "description": "List of results of the test case.",
+      "type": "array",
+      "items": {
+        "$ref": "./basic.json#/definitions/testCaseResult"
+      }
+    },
+    "owner": {
+      "description": "Owner of this Pipeline.",
+      "$ref": "../type/entityReference.json",
+      "default": null
+    },
+    "updatedAt": {
+      "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
+      "$ref": "../type/basic.json#/definitions/timestamp"
+    },
+    "updatedBy": {
+      "description": "User who made the update.",
+      "type": "string"
+    },
+    "deleted": {
+      "description": "When `true` indicates the entity has been soft deleted.",
+      "type": "boolean",
+      "default": false
+    }
+  },
+  "required": ["name", "column", "testCase"],
+  "additionalProperties": false
+}

catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json

@@ -0,0 +1,16 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/tableColumnCountToEqual.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "TableRowCountToEqual",
+  "description": "This scheam defines the test TableColumnCountToEqual. Test the number of columns equal to a value.",
+  "type": "object",
+  "javaType": "org.openmetadata.catalog.tests.table.TableColumnCountToEqual",
+  "properties": {
+    "value": {
+      "description": "Expected number of columns to equal to a  {value}",
+      "type": "integer"
+    }
+  },
+  "required": ["value"],
+  "additionalProperties": false
+}

catalog-rest-service/src/main/resources/json/schema/tests/table/tableRowCountToBeBetween.json

@@ -0,0 +1,20 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/tableRowCountToBeBetween.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "TableRowCountToEqual",
+  "description": "This scheam defines the test TableRowCountToBeBetween. Test the number of rows to between to two values.",
+  "type": "object",
+  "javaType": "org.openmetadata.catalog.tests.table.TableRowCountToBeBetween",
+  "properties": {
+    "minValue": {
+      "description": "Expected number of rows should be greater than or equal to {minValue}. If minValue is not included, maxValue is treated as upperBound and there will be no minimum number of rows",
+      "type": "integer"
+    },
+    "maxValue": {
+      "description": "Expected number of rows should be lower than or equal to {maxValue}. if maxValue is not included, minValue is treated as lowerBound and there will eb no maximum number of rows",
+      "type": "integer"
+    }
+  },
+  "anyOf": [{ "required": ["minValue"] }, { "required": ["maxValue"] }],
+  "additionalProperties": false
+}

catalog-rest-service/src/main/resources/json/schema/tests/table/tableRowCountToEqual.json

@@ -0,0 +1,16 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/table/tableRowCountToEqual.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "TableRowCountToEqual",
+  "description": "This schema defines the test TableRowCountToEqual. Test the number of rows equal to a value.",
+  "type": "object",
+  "javaType": "org.openmetadata.catalog.tests.table.TableRowCountToEqual",
+  "properties": {
+    "value": {
+      "description": "Expected number of rows {value}",
+      "type": "integer"
+    }
+  },
+  "required": ["value"],
+  "additionalProperties": false
+}

catalog-rest-service/src/main/resources/json/schema/tests/tableTest.json

@@ -0,0 +1,90 @@
+{
+  "$id": "https://open-metadata.org/schema/tests/tableTest.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "TableTest",
+  "description": "TableTest is a test definition to capture data quality tests against tables and columns.",
+  "type": "object",
+  "javaType": "org.openmetadata.catalog.tests.TableTest",
+  "definitions": {
+    "tableTestCase": {
+      "description": "Table Test Case.",
+      "type": "object",
+      "properties": {
+        "config": {
+          "oneOf": [
+            {
+              "$ref": "./table/tableRowCountToEqual.json"
+            },
+            {
+              "$ref": "./table/tableRowCountToBeBetween.json"
+            },
+            {
+              "$ref": "./table/tableColumnCountToEqual.json"
+            }
+          ]
+        },
+        "testType": {
+          "enum": [
+            "tableRowCountToEqual",
+            "tableRowCountToBeBetween",
+            "tableColumnCountToEqual"
+          ]
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "properties": {
+    "id": {
+      "description": "Unique identifier of this table instance.",
+      "$ref": "../type/basic.json#/definitions/uuid"
+    },
+    "name": {
+      "description": "Name that identifies this test case.",
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 128
+    },
+    "description": {
+      "description": "Description of the testcase.",
+      "type": "string"
+    },
+    "tableName": {
+      "description": "Table Name for which this test applies.",
+      "type": "string"
+    },
+    "tableTestCase": {
+      "$ref": "#/definitions/tableTestCase"
+    },
+    "executionFrequency": {
+      "$ref": "./basic.json#/definitions/testCaseExecutionFrequency"
+    },
+    "results": {
+      "description": "List of results of the test case.",
+      "type": "array",
+      "items": {
+        "$ref": "./basic.json#/definitions/testCaseResult"
+      }
+    },
+    "owner": {
+      "description": "Owner of this Pipeline.",
+      "$ref": "../type/entityReference.json",
+      "default": null
+    },
+    "updatedAt": {
+      "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
+      "$ref": "../type/basic.json#/definitions/timestamp"
+    },
+    "updatedBy": {
+      "description": "User who made the update.",
+      "type": "string"
+    },
+    "deleted": {
+      "description": "When `true` indicates the entity has been soft deleted.",
+      "type": "boolean",
+      "default": false
+    }
+  },
+  "required": ["name", "testCase", "tableName"],
+  "additionalProperties": false
+}

catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java

@@ -85,6 +85,21 @@ import org.openmetadata.catalog.resources.databases.TableResource.TableList;
 import org.openmetadata.catalog.resources.locations.LocationResourceTest;
 import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest;
 import org.openmetadata.catalog.resources.tags.TagResourceTest;
+import org.openmetadata.catalog.tests.ColumnTest;
+import org.openmetadata.catalog.tests.ColumnTestCase;
+import org.openmetadata.catalog.tests.TableTest;
+import org.openmetadata.catalog.tests.TableTestCase;
+import org.openmetadata.catalog.tests.column.ColumnValueLengthsToBeBetween;
+import org.openmetadata.catalog.tests.column.ColumnValuesMissingCountToBeEqual;
+import org.openmetadata.catalog.tests.column.ColumnValuesToBeNotNull;
+import org.openmetadata.catalog.tests.column.ColumnValuesToBeUnique;
+import org.openmetadata.catalog.tests.column.ColumnValuesToMatchRegex;
+import org.openmetadata.catalog.tests.table.TableColumnCountToEqual;
+import org.openmetadata.catalog.tests.table.TableRowCountToBeBetween;
+import org.openmetadata.catalog.tests.table.TableRowCountToEqual;
+import org.openmetadata.catalog.tests.type.TestCaseExecutionFrequency;
+import org.openmetadata.catalog.tests.type.TestCaseResult;
+import org.openmetadata.catalog.tests.type.TestCaseStatus;
 import org.openmetadata.catalog.type.ChangeDescription;
 import org.openmetadata.catalog.type.Column;
 import org.openmetadata.catalog.type.ColumnConstraint;
@@ -1019,6 +1034,119 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
     assertEquals(expected.getGeneratedAt(), actual.getGeneratedAt());
   }
 
+  @Test
+  void put_tableColumnTests_200(TestInfo test) throws IOException {
+    Table table = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS);
+    TableRowCountToEqual tableRowCountToEqual = new TableRowCountToEqual().withValue(100);
+    TableTestCase tableTestCase =
+        new TableTestCase()
+            .withTestType(TableTestCase.TestType.TABLE_ROW_COUNT_TO_EQUAL)
+            .withConfig(tableRowCountToEqual);
+    TableTest tableTest =
+        new TableTest()
+            .withName("test1")
+            .withTableTestCase(tableTestCase)
+            .withExecutionFrequency(TestCaseExecutionFrequency.Hourly);
+    Table putResponse = putTableTest(table.getId(), tableTest, ADMIN_AUTH_HEADERS);
+    verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(tableTest));
+
+    table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS);
+    verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest));
+
+    // Add result to tableTest
+    TestCaseResult testCaseResult1 =
+        new TestCaseResult()
+            .withResult("Rows equal to 100")
+            .withStatus(TestCaseStatus.Success)
+            .withSampleData("Rows == 100")
+            .withExecutionTime(100L);
+    tableTest.setResults(List.of(testCaseResult1));
+    tableTest.setId(table.getTableTests().get(0).getId());
+    putResponse = putTableTest(table.getId(), tableTest, ADMIN_AUTH_HEADERS);
+    verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(tableTest));
+
+    TestCaseResult testCaseResult2 =
+        new TestCaseResult()
+            .withResult("Rows equal to 100")
+            .withStatus(TestCaseStatus.Success)
+            .withSampleData("Rows == 100")
+            .withExecutionTime(100L);
+    tableTest.setResults(List.of(testCaseResult2));
+    tableTest.setId(table.getTableTests().get(0).getId());
+
+    table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS);
+    verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest));
+    TableRowCountToBeBetween tableRowCountToBeBetween =
+        new TableRowCountToBeBetween().withMinValue(100).withMaxValue(1000);
+    TableTestCase tableTestCase1 =
+        new TableTestCase()
+            .withTestType(TableTestCase.TestType.TABLE_ROW_COUNT_TO_BE_BETWEEN)
+            .withConfig(tableRowCountToBeBetween);
+    TableTest tableTest1 = new TableTest().withName("column_value_to_be_unique").withTableTestCase(tableTestCase1);
+    putResponse = putTableTest(table.getId(), tableTest1, ADMIN_AUTH_HEADERS);
+    verifyTableTest(putResponse.getName(), putResponse.getTableTests(), List.of(tableTest, tableTest1));
+    table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS);
+    verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest, tableTest1));
+
+    Column c1 = table.getColumns().get(0);
+    ColumnValueLengthsToBeBetween columnValueLengthsToBeBetween =
+        new ColumnValueLengthsToBeBetween().withMaxValue(100).withMinValue(10);
+    ColumnTestCase columnTestCase =
+        new ColumnTestCase()
+            .withTestType(ColumnTestCase.TestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN)
+            .withConfig(columnValueLengthsToBeBetween);
+    ColumnTest columnTest =
+        new ColumnTest()
+            .withColumnName(c1.getName())
+            .withName("test")
+            .withTestCase(columnTestCase)
+            .withExecutionFrequency(TestCaseExecutionFrequency.Hourly);
+    putResponse = putColumnTest(table.getId(), columnTest, ADMIN_AUTH_HEADERS);
+    verifyColumnTest(putResponse, c1, List.of(columnTest));
+
+    table = getEntity(table.getId(), "tests", ADMIN_AUTH_HEADERS);
+    verifyTableTest(table.getName(), table.getTableTests(), List.of(tableTest, tableTest1));
+    verifyColumnTest(table, c1, List.of(columnTest));
+
+    // Add result to columnTest
+    TestCaseResult colTestCaseResult =
+        new TestCaseResult()
+            .withResult("min is > 100 and max < 1000")
+            .withStatus(TestCaseStatus.Success)
+            .withSampleData("minValue is 100 and maxValue is 1000")
+            .withExecutionTime(100L);
+    columnTest.setResults(List.of(colTestCaseResult));
+    putResponse = putColumnTest(table.getId(), columnTest, ADMIN_AUTH_HEADERS);
+    verifyColumnTest(putResponse, c1, List.of(columnTest));
+
+    ColumnValuesMissingCountToBeEqual columnValuesMissingCountToBeEqual =
+        new ColumnValuesMissingCountToBeEqual().withMissingCountValue(10);
+    ColumnTestCase columnTestCase1 =
+        new ColumnTestCase()
+            .withTestType(ColumnTestCase.TestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL)
+            .withConfig(columnValuesMissingCountToBeEqual);
+    ColumnTest columnTest1 =
+        new ColumnTest()
+            .withColumnName(c1.getName())
+            .withName("test")
+            .withTestCase(columnTestCase1)
+            .withExecutionFrequency(TestCaseExecutionFrequency.Hourly);
+    putResponse = putColumnTest(table.getId(), columnTest1, ADMIN_AUTH_HEADERS);
+    verifyColumnTest(putResponse, c1, List.of(columnTest, columnTest1));
+
+    // Add result to columnTest
+    TestCaseResult colTestCaseResult1 =
+        new TestCaseResult()
+            .withResult("min is > 100 and max < 1000")
+            .withStatus(TestCaseStatus.Success)
+            .withSampleData("minValue is 100 and maxValue is 1000")
+            .withExecutionTime(100L);
+    columnTest.setResults(List.of(colTestCaseResult1));
+    putResponse = putColumnTest(table.getId(), columnTest, ADMIN_AUTH_HEADERS);
+    columnTest.setResults(List.of(colTestCaseResult, colTestCaseResult1));
+    verifyColumnTest(putResponse, c1, List.of(columnTest, columnTest1));
+  }
+
   @Test
   void get_deletedTableWithDeleteLocation(TestInfo test) throws IOException {
     CreateTable create = createRequest(getEntityName(test), "description", "displayName", USER_OWNER1);
@@ -1500,6 +1628,18 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
     return TestUtils.put(target, dataModel, Table.class, OK, authHeaders);
   }
 
+  public static Table putTableTest(UUID tableId, TableTest data, Map<String, String> authHeaders)
+      throws HttpResponseException {
+    WebTarget target = CatalogApplicationTest.getResource("tables/" + tableId + "/tableTest");
+    return TestUtils.put(target, data, Table.class, OK, authHeaders);
+  }
+
+  public static Table putColumnTest(UUID tableId, ColumnTest data, Map<String, String> authHeaders)
+      throws HttpResponseException {
+    WebTarget target = CatalogApplicationTest.getResource("tables/" + tableId + "/columnTest");
+    return TestUtils.put(target, data, Table.class, OK, authHeaders);
+  }
+
   private static int getTagUsageCount(String tagFQN, Map<String, String> authHeaders) throws HttpResponseException {
     return TagResourceTest.getTag(tagFQN, "usageCount", authHeaders).getUsageCount();
   }
@@ -1522,6 +1662,118 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
     }
   }
 
+  private void verifyTableTest(String tableName, List<TableTest> actualTests, List<TableTest> expectedTests)
+      throws IOException {
+    assertEquals(actualTests.size(), expectedTests.size());
+    Map<String, TableTest> tableTestMap = new HashMap<>();
+    for (TableTest test : actualTests) {
+      tableTestMap.put(test.getName(), test);
+    }
+    for (TableTest test : expectedTests) {
+      // passed in test name will be overridden in backend
+      String expectedTestName = tableName + "." + test.getTableTestCase().getTestType().toString();
+      TableTest storedTest = tableTestMap.get(expectedTestName);
+      assertNotNull(storedTest);
+      assertEquals(expectedTestName, storedTest.getName());
+      assertEquals(test.getDescription(), storedTest.getDescription());
+      assertEquals(test.getExecutionFrequency(), storedTest.getExecutionFrequency());
+      assertEquals(test.getOwner(), storedTest.getOwner());
+      verifyTableTestCase(test.getTableTestCase(), storedTest.getTableTestCase());
+      verifyTestCaseResults(test.getResults(), storedTest.getResults());
+    }
+  }
+
+  private void verifyTableTestCase(TableTestCase expected, TableTestCase actual) {
+    assertEquals(expected.getTestType(), actual.getTestType());
+    if (expected.getTestType() == TableTestCase.TestType.TABLE_COLUMN_COUNT_TO_EQUAL) {
+      TableColumnCountToEqual expectedTest = (TableColumnCountToEqual) expected.getConfig();
+      TableColumnCountToEqual actualTest = JsonUtils.convertValue(actual.getConfig(), TableColumnCountToEqual.class);
+      assertEquals(expectedTest.getValue(), actualTest.getValue());
+    } else if (expected.getTestType() == TableTestCase.TestType.TABLE_ROW_COUNT_TO_BE_BETWEEN) {
+      TableRowCountToBeBetween expectedTest = (TableRowCountToBeBetween) expected.getConfig();
+      TableRowCountToBeBetween actualTest = JsonUtils.convertValue(actual.getConfig(), TableRowCountToBeBetween.class);
+      assertEquals(expectedTest.getMaxValue(), actualTest.getMaxValue());
+      assertEquals(expectedTest.getMinValue(), actualTest.getMinValue());
+    } else if (expected.getTestType() == TableTestCase.TestType.TABLE_ROW_COUNT_TO_EQUAL) {
+      TableRowCountToEqual expectedTest = (TableRowCountToEqual) expected.getConfig();
+      TableRowCountToEqual actualTest = JsonUtils.convertValue(actual.getConfig(), TableRowCountToEqual.class);
+      assertEquals(expectedTest.getValue(), actualTest.getValue());
+    }
+  }
+
+  private void verifyColumnTest(Table table, Column column, List<ColumnTest> expectedTests) throws IOException {
+    List<ColumnTest> actualTests = new ArrayList<>();
+    for (Column c : table.getColumns()) {
+      if (c.getName().equals(column.getName())) {
+        actualTests = c.getColumnTests();
+      }
+    }
+    assertEquals(actualTests.size(), expectedTests.size());
+
+    Map<String, ColumnTest> columnTestMap = new HashMap<>();
+    for (ColumnTest test : actualTests) {
+      columnTestMap.put(test.getName(), test);
+    }
+
+    for (ColumnTest test : expectedTests) {
+      // passed in test name will be overridden in backend
+      String expectedTestName = column.getName() + "." + test.getTestCase().getTestType().toString();
+      ColumnTest storedTest = columnTestMap.get(expectedTestName);
+      assertNotNull(storedTest);
+      assertEquals(expectedTestName, storedTest.getName());
+      assertEquals(test.getDescription(), storedTest.getDescription());
+      assertEquals(test.getExecutionFrequency(), storedTest.getExecutionFrequency());
+      assertEquals(test.getOwner(), storedTest.getOwner());
+      verifyColumnTestCase(test.getTestCase(), storedTest.getTestCase());
+      verifyTestCaseResults(test.getResults(), storedTest.getResults());
+    }
+  }
+
+  private void verifyColumnTestCase(ColumnTestCase expected, ColumnTestCase actual) {
+    assertEquals(expected.getTestType(), actual.getTestType());
+    if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_TO_BE_UNIQUE) {
+      ColumnValuesToBeUnique expectedTest = (ColumnValuesToBeUnique) expected.getConfig();
+      ColumnValuesToBeUnique actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesToBeUnique.class);
+      assertEquals(expectedTest, actualTest);
+    } else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_TO_BE_NOT_NULL) {
+      ColumnValuesToBeNotNull expectedTest = (ColumnValuesToBeNotNull) expected.getConfig();
+      ColumnValuesToBeNotNull actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesToBeNotNull.class);
+      assertEquals(expectedTest, actualTest);
+    } else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_TO_MATCH_REGEX) {
+      ColumnValuesToMatchRegex expectedTest = (ColumnValuesToMatchRegex) expected.getConfig();
+      ColumnValuesToMatchRegex actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValuesToMatchRegex.class);
+      assertEquals(expectedTest.getRegex(), actualTest.getRegex());
+    } else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN) {
+      ColumnValueLengthsToBeBetween expectedTest = (ColumnValueLengthsToBeBetween) expected.getConfig();
+      ColumnValueLengthsToBeBetween actualTest =
+          JsonUtils.convertValue(actual.getConfig(), ColumnValueLengthsToBeBetween.class);
+      assertEquals(expectedTest.getMaxValue(), actualTest.getMaxValue());
+      assertEquals(expectedTest.getMinValue(), actualTest.getMinValue());
+    } else if (expected.getTestType() == ColumnTestCase.TestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL) {
+      ColumnValuesMissingCountToBeEqual expectedTest = (ColumnValuesMissingCountToBeEqual) expected.getConfig();
+      ColumnValuesMissingCountToBeEqual actualTest =
+          JsonUtils.convertValue(actual.getConfig(), ColumnValuesMissingCountToBeEqual.class);
+      assertEquals(expectedTest.getMissingCountValue(), actualTest.getMissingCountValue());
+      assertEquals(expectedTest.getMissingValueMatch(), actualTest.getMissingValueMatch());
+    }
+  }
+
+  private void verifyTestCaseResults(List<TestCaseResult> expected, List<TestCaseResult> actual) throws IOException {
+    assertEquals(expected.size(), actual.size());
+    Map<Long, TestCaseResult> actualResultMap = new HashMap<>();
+    for (Object a : actual) {
+      TestCaseResult result = JsonUtils.convertValue(a, TestCaseResult.class);
+      actualResultMap.put(result.getExecutionTime(), result);
+    }
+    for (Object e : expected) {
+      TestCaseResult result = JsonUtils.convertValue(e, TestCaseResult.class);
+      TestCaseResult actualResult = actualResultMap.get(result.getExecutionTime());
+      assertNotNull(actualResult);
+      assertEquals(result.getResult(), actualResult.getResult());
+      assertEquals(result.getSampleData(), actualResult.getSampleData());
+    }
+  }
+
   @Override
   public CreateTable createRequest(String name, String description, String displayName, EntityReference owner) {
     TableConstraint constraint =