Fix #3217 - Test case props should be unique (#3220)

Fix #3217 - Test case props should be unique (#3220)

catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesLengthsToBeBetween.json

@@ -6,21 +6,21 @@
   "type": "object",
   "javaType": "org.openmetadata.catalog.tests.column.ColumnValueLengthsToBeBetween",
   "properties": {
-    "minValue": {
-      "description": "The {minValue} for the column length. If minValue is not included, maxValue is treated as upperBound and there will be no minimum number of rows",
+    "minLength": {
+      "description": "The {minLength} for the column length. If minLength is not included, maxLength is treated as upperBound and there will be no minimum number of rows",
       "type": "integer"
     },
-    "maxValue": {
-      "description": "The {maxValue} for the column length. if maxValue is not included, minValue is treated as lowerBound and there will eb no maximum number of rows",
+    "maxLength": {
+      "description": "The {maxLength} for the column length. if maxLength is not included, minLength is treated as lowerBound and there will eb no maximum number of rows",
       "type": "integer"
     }
   },
   "anyOf": [
     {
-      "required": ["minValue"]
+      "required": ["minLength"]
     },
     {
-      "required": ["maxValue"]
+      "required": ["maxLength"]
     }
   ],
   "additionalProperties": false

catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json

@@ -6,11 +6,11 @@
   "type": "object",
   "javaType": "org.openmetadata.catalog.tests.table.TableColumnCountToEqual",
   "properties": {
-    "value": {
-      "description": "Expected number of columns to equal to a  {value}",
+    "columnCount": {
+      "description": "Expected number of columns to equal to a {value}",
       "type": "integer"
     }
   },
-  "required": ["value"],
+  "required": ["columnCount"],
   "additionalProperties": false
 }

catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java

@@ -1093,7 +1093,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
 
     Column c1 = table.getColumns().get(0);
     ColumnValueLengthsToBeBetween columnValueLengthsToBeBetween =
-        new ColumnValueLengthsToBeBetween().withMaxValue(100).withMinValue(10);
+        new ColumnValueLengthsToBeBetween().withMaxLength(100).withMinLength(10);
     ColumnTestCase columnTestCase =
         new ColumnTestCase()
             .withColumnTestType(ColumnTestCase.ColumnTestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN)
@@ -1746,7 +1746,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
     if (expected.getTableTestType() == TableTestCase.TableTestType.TABLE_COLUMN_COUNT_TO_EQUAL) {
       TableColumnCountToEqual expectedTest = (TableColumnCountToEqual) expected.getConfig();
       TableColumnCountToEqual actualTest = JsonUtils.convertValue(actual.getConfig(), TableColumnCountToEqual.class);
-      assertEquals(expectedTest.getValue(), actualTest.getValue());
+      assertEquals(expectedTest.getColumnCount(), actualTest.getColumnCount());
     } else if (expected.getTableTestType() == TableTestCase.TableTestType.TABLE_ROW_COUNT_TO_BE_BETWEEN) {
       TableRowCountToBeBetween expectedTest = (TableRowCountToBeBetween) expected.getConfig();
       TableRowCountToBeBetween actualTest = JsonUtils.convertValue(actual.getConfig(), TableRowCountToBeBetween.class);
@@ -1807,8 +1807,8 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
       ColumnValueLengthsToBeBetween expectedTest = (ColumnValueLengthsToBeBetween) expected.getConfig();
       ColumnValueLengthsToBeBetween actualTest =
           JsonUtils.convertValue(actual.getConfig(), ColumnValueLengthsToBeBetween.class);
-      assertEquals(expectedTest.getMaxValue(), actualTest.getMaxValue());
-      assertEquals(expectedTest.getMinValue(), actualTest.getMinValue());
+      assertEquals(expectedTest.getMaxLength(), actualTest.getMaxLength());
+      assertEquals(expectedTest.getMinLength(), actualTest.getMinLength());
     } else if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL) {
       ColumnValuesMissingCountToBeEqual expectedTest = (ColumnValuesMissingCountToBeEqual) expected.getConfig();
       ColumnValuesMissingCountToBeEqual actualTest =

ingestion-core/src/metadata/_version.py

@@ -7,5 +7,5 @@ Provides metadata version information.
 
 from incremental import Version
 
-__version__ = Version("metadata", 0, 9, 0, dev=20)
+__version__ = Version("metadata", 0, 9, 0, dev=21)
 __all__ = ["__version__"]

ingestion/src/metadata/orm_profiler/validations/column/column_values_length_to_be_between.py

@@ -56,13 +56,13 @@ def column_value_length_to_be_between(
 
     status = (
         TestCaseStatus.Success
-        if col_profile.minLength >= test_case.minValue
-        and col_profile.maxLength <= test_case.maxValue
+        if col_profile.minLength >= test_case.minLength
+        and col_profile.maxLength <= test_case.maxLength
         else TestCaseStatus.Failed
     )
     result = (
         f"Found minLength={col_profile.minLength}, maxLength={col_profile.maxLength} vs."
-        + f" the expected minLength={test_case.minValue}, maxLength={test_case.maxValue}."
+        + f" the expected minLength={test_case.minLength}, maxLength={test_case.maxLength}."
     )
 
     return TestCaseResult(

ingestion/src/metadata/orm_profiler/validations/table/table_column_count_to_equal.py

@@ -49,12 +49,10 @@ def table_column_count_to_equal(
 
     status = (
         TestCaseStatus.Success
-        if table_profile.columnCount == test_case.value
+        if table_profile.columnCount == test_case.columnCount
         else TestCaseStatus.Failed
     )
-    result = (
-        f"Found {table_profile.columnCount} columns vs. the expected {test_case.value}"
-    )
+    result = f"Found {table_profile.columnCount} columns vs. the expected {test_case.columnCount}"
 
     return TestCaseResult(
         executionTime=execution_date.timestamp(), testCaseStatus=status, result=result

ingestion/tests/unit/profiler/test_validation_parsing.py

@@ -0,0 +1,182 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Test that we can properly parse JSON definition
+of validations
+"""
+from metadata.generated.schema.tests.column.columnValuesLengthsToBeBetween import (
+    ColumnValueLengthsToBeBetween,
+)
+from metadata.generated.schema.tests.column.columnValuesMissingCountToBeEqual import (
+    ColumnValuesMissingCount,
+)
+from metadata.generated.schema.tests.column.columnValuesToBeBetween import (
+    ColumnValuesToBeBetween,
+)
+from metadata.generated.schema.tests.column.columnValuesToBeNotInSet import (
+    ColumnValuesToBeNotInSet,
+)
+from metadata.generated.schema.tests.column.columnValuesToBeNotNull import (
+    ColumnValuesToBeNotNull,
+)
+from metadata.generated.schema.tests.column.columnValuesToBeUnique import (
+    ColumnValuesToBeUnique,
+)
+from metadata.generated.schema.tests.column.columnValuesToMatchRegex import (
+    ColumnValuesToMatchRegex,
+)
+from metadata.generated.schema.tests.columnTest import ColumnTestCase
+from metadata.generated.schema.tests.table.tableColumnCountToEqual import (
+    TableColumnCountToEqual,
+)
+from metadata.generated.schema.tests.table.tableRowCountToBeBetween import (
+    TableRowCountToBeBetween,
+)
+from metadata.generated.schema.tests.table.tableRowCountToEqual import (
+    TableRowCountToEqual,
+)
+from metadata.generated.schema.tests.tableTest import TableTestCase
+
+
+def test_column_values_to_be_unique():
+    """
+    ColumnValuesToBeUnique
+    """
+    obj = {"config": {}, "columnTestType": "columnValuesToBeUnique"}
+
+    test_case = ColumnTestCase.parse_obj(obj)
+
+    assert isinstance(test_case.config, ColumnValuesToBeUnique)
+
+
+def test_column_values_to_be_not_null():
+    """
+    ColumnValuesToBeNotNull
+    """
+    obj = {"config": {}, "columnTestType": "columnValuesToBeNotNull"}
+
+    test_case = ColumnTestCase.parse_obj(obj)
+
+    # TODO: we should parse this properly
+    # assert isinstance(test_case.config, ColumnValuesToBeNotNull)
+
+
+def test_column_values_to_be_between():
+    """
+    ColumnValuesToBeBetween
+    """
+    obj = {
+        "config": {"minValue": 6, "maxValue": 10},
+        "columnTestType": "columnValuesToBeBetween",
+    }
+
+    test_case = ColumnTestCase.parse_obj(obj)
+
+    assert isinstance(test_case.config, ColumnValuesToBeBetween)
+
+
+def test_column_value_length_to_be_between():
+    """
+    ColumnValueLengthsToBeBetween
+    """
+    obj = {
+        "config": {"minLength": 6, "maxLength": 10},
+        "columnTestType": "columnValueLengthsToBeBetween",
+    }
+
+    test_case = ColumnTestCase.parse_obj(obj)
+
+    assert isinstance(test_case.config, ColumnValueLengthsToBeBetween)
+
+
+def test_column_values_not_in_set():
+    """
+    ColumnValuesToBeNotInSet
+    """
+    obj = {
+        "config": {"forbiddenValues": ["random"]},
+        "columnTestType": "columnValuesToBeNotInSet",
+    }
+
+    test_case = ColumnTestCase.parse_obj(obj)
+
+    assert isinstance(test_case.config, ColumnValuesToBeNotInSet)
+
+
+def test_column_values_to_match_regex():
+    """
+    ColumnValuesToMatchRegex
+    """
+    obj = {
+        "config": {"regex": "%regex%"},
+        "columnTestType": "columnValuesToMatchRegex",
+    }
+
+    test_case = ColumnTestCase.parse_obj(obj)
+
+    assert isinstance(test_case.config, ColumnValuesToMatchRegex)
+
+
+def test_column_values_missing_count_to_be_equal():
+    """
+    ColumnValuesMissingCount
+    """
+    obj = {
+        "config": {"missingCountValue": 10, "missingValueMatch": ["N/A"]},
+        "columnTestType": "columnValuesMissingCountToBeEqual",
+    }
+
+    test_case = ColumnTestCase.parse_obj(obj)
+
+    assert isinstance(test_case.config, ColumnValuesMissingCount)
+
+
+def test_table_row_count_to_equal():
+    """
+    TableRowCountToEqual
+    """
+    obj = {
+        "config": {"value": 10},
+        "tableTestType": "tableRowCountToEqual",
+    }
+
+    test_case = TableTestCase.parse_obj(obj)
+
+    assert isinstance(test_case.config, TableRowCountToEqual)
+
+
+def test_table_row_count_to_be_between():
+    """
+    TableRowCountToBeBetween
+    """
+    obj = {
+        "config": {"minValue": 10, "maxValue": 100},
+        "tableTestType": "tableRowCountToBeBetween",
+    }
+
+    test_case = TableTestCase.parse_obj(obj)
+
+    assert isinstance(test_case.config, TableRowCountToBeBetween)
+
+
+def test_table_column_count_to_equal():
+    """
+    TableColumnCountToEqual
+    """
+    obj = {
+        "config": {"columnCount": 10},
+        "tableTestType": "tableColumnCountToEqual",
+    }
+
+    test_case = TableTestCase.parse_obj(obj)
+
+    assert isinstance(test_case.config, TableColumnCountToEqual)

ingestion/tests/unit/profiler/test_validations.py

@@ -153,7 +153,7 @@ def test_table_column_count_to_equal():
     )
 
     res_ok = validate(
-        TableColumnCountToEqual(value=5),
+        TableColumnCountToEqual(columnCount=5),
         table_profile=table_profile,
         execution_date=EXECUTION_DATE,
     )
@@ -164,7 +164,7 @@ def test_table_column_count_to_equal():
     )
 
     res_ko = validate(
-        TableColumnCountToEqual(value=20),
+        TableColumnCountToEqual(columnCount=20),
         table_profile=table_profile,
         execution_date=EXECUTION_DATE,
     )
@@ -180,7 +180,7 @@ def test_table_column_count_to_equal():
     )
 
     res_aborted = validate(
-        TableColumnCountToEqual(value=5),
+        TableColumnCountToEqual(columnCount=5),
         table_profile=table_profile_aborted,
         execution_date=EXECUTION_DATE,
     )
@@ -379,7 +379,7 @@ def test_column_value_length_to_be_between():
     )
 
     res_ok = validate(
-        ColumnValueLengthsToBeBetween(minValue=2, maxValue=20),
+        ColumnValueLengthsToBeBetween(minLength=2, maxLength=20),
         col_profile=col_profile,
         execution_date=EXECUTION_DATE,
     )
@@ -390,7 +390,7 @@ def test_column_value_length_to_be_between():
     )
 
     res_ko = validate(
-        ColumnValueLengthsToBeBetween(minValue=10, maxValue=20),
+        ColumnValueLengthsToBeBetween(minLength=10, maxLength=20),
         col_profile=col_profile,
         execution_date=EXECUTION_DATE,
     )
@@ -404,7 +404,7 @@ def test_column_value_length_to_be_between():
     col_profile_aborted = ColumnProfile(minLength=4)
 
     res_aborted = validate(
-        ColumnValueLengthsToBeBetween(minValue=2, maxValue=20),
+        ColumnValueLengthsToBeBetween(minLength=2, maxLength=20),
         col_profile=col_profile_aborted,
         execution_date=EXECUTION_DATE,
     )