diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesLengthsToBeBetween.json b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesLengthsToBeBetween.json index fb262ec5369..d28ed4ba21b 100644 --- a/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesLengthsToBeBetween.json +++ b/catalog-rest-service/src/main/resources/json/schema/tests/column/columnValuesLengthsToBeBetween.json @@ -6,21 +6,21 @@ "type": "object", "javaType": "org.openmetadata.catalog.tests.column.ColumnValueLengthsToBeBetween", "properties": { - "minValue": { - "description": "The {minValue} for the column length. If minValue is not included, maxValue is treated as upperBound and there will be no minimum number of rows", + "minLength": { + "description": "The {minLength} for the column length. If minLength is not included, maxLength is treated as upperBound and there will be no minimum number of rows", "type": "integer" }, - "maxValue": { - "description": "The {maxValue} for the column length. if maxValue is not included, minValue is treated as lowerBound and there will eb no maximum number of rows", + "maxLength": { + "description": "The {maxLength} for the column length. if maxLength is not included, minLength is treated as lowerBound and there will eb no maximum number of rows", "type": "integer" } }, "anyOf": [ { - "required": ["minValue"] + "required": ["minLength"] }, { - "required": ["maxValue"] + "required": ["maxLength"] } ], "additionalProperties": false diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json index a139d860ac1..8b82c4b9f7f 100644 --- a/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json +++ b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnCountToEqual.json @@ -6,11 +6,11 @@ "type": "object", "javaType": "org.openmetadata.catalog.tests.table.TableColumnCountToEqual", "properties": { - "value": { - "description": "Expected number of columns to equal to a {value}", + "columnCount": { + "description": "Expected number of columns to equal to a {value}", "type": "integer" } }, - "required": ["value"], + "required": ["columnCount"], "additionalProperties": false } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java index d7c074c5408..00adeb4a7af 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java @@ -1093,7 +1093,7 @@ public class TableResourceTest extends EntityResourceTest { Column c1 = table.getColumns().get(0); ColumnValueLengthsToBeBetween columnValueLengthsToBeBetween = - new ColumnValueLengthsToBeBetween().withMaxValue(100).withMinValue(10); + new ColumnValueLengthsToBeBetween().withMaxLength(100).withMinLength(10); ColumnTestCase columnTestCase = new ColumnTestCase() .withColumnTestType(ColumnTestCase.ColumnTestType.COLUMN_VALUE_LENGTHS_TO_BE_BETWEEN) @@ -1746,7 +1746,7 @@ public class TableResourceTest extends EntityResourceTest { if (expected.getTableTestType() == TableTestCase.TableTestType.TABLE_COLUMN_COUNT_TO_EQUAL) { TableColumnCountToEqual expectedTest = (TableColumnCountToEqual) expected.getConfig(); TableColumnCountToEqual actualTest = JsonUtils.convertValue(actual.getConfig(), TableColumnCountToEqual.class); - assertEquals(expectedTest.getValue(), actualTest.getValue()); + assertEquals(expectedTest.getColumnCount(), actualTest.getColumnCount()); } else if (expected.getTableTestType() == TableTestCase.TableTestType.TABLE_ROW_COUNT_TO_BE_BETWEEN) { TableRowCountToBeBetween expectedTest = (TableRowCountToBeBetween) expected.getConfig(); TableRowCountToBeBetween actualTest = JsonUtils.convertValue(actual.getConfig(), TableRowCountToBeBetween.class); @@ -1807,8 +1807,8 @@ public class TableResourceTest extends EntityResourceTest { ColumnValueLengthsToBeBetween expectedTest = (ColumnValueLengthsToBeBetween) expected.getConfig(); ColumnValueLengthsToBeBetween actualTest = JsonUtils.convertValue(actual.getConfig(), ColumnValueLengthsToBeBetween.class); - assertEquals(expectedTest.getMaxValue(), actualTest.getMaxValue()); - assertEquals(expectedTest.getMinValue(), actualTest.getMinValue()); + assertEquals(expectedTest.getMaxLength(), actualTest.getMaxLength()); + assertEquals(expectedTest.getMinLength(), actualTest.getMinLength()); } else if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUES_MISSING_COUNT_TO_BE_EQUAL) { ColumnValuesMissingCountToBeEqual expectedTest = (ColumnValuesMissingCountToBeEqual) expected.getConfig(); ColumnValuesMissingCountToBeEqual actualTest = diff --git a/ingestion-core/src/metadata/_version.py b/ingestion-core/src/metadata/_version.py index d8870de4bd2..315c1541753 100644 --- a/ingestion-core/src/metadata/_version.py +++ b/ingestion-core/src/metadata/_version.py @@ -7,5 +7,5 @@ Provides metadata version information. from incremental import Version -__version__ = Version("metadata", 0, 9, 0, dev=20) +__version__ = Version("metadata", 0, 9, 0, dev=21) __all__ = ["__version__"] diff --git a/ingestion/src/metadata/orm_profiler/validations/column/column_values_length_to_be_between.py b/ingestion/src/metadata/orm_profiler/validations/column/column_values_length_to_be_between.py index 27220896275..ae90a795eca 100644 --- a/ingestion/src/metadata/orm_profiler/validations/column/column_values_length_to_be_between.py +++ b/ingestion/src/metadata/orm_profiler/validations/column/column_values_length_to_be_between.py @@ -56,13 +56,13 @@ def column_value_length_to_be_between( status = ( TestCaseStatus.Success - if col_profile.minLength >= test_case.minValue - and col_profile.maxLength <= test_case.maxValue + if col_profile.minLength >= test_case.minLength + and col_profile.maxLength <= test_case.maxLength else TestCaseStatus.Failed ) result = ( f"Found minLength={col_profile.minLength}, maxLength={col_profile.maxLength} vs." - + f" the expected minLength={test_case.minValue}, maxLength={test_case.maxValue}." + + f" the expected minLength={test_case.minLength}, maxLength={test_case.maxLength}." ) return TestCaseResult( diff --git a/ingestion/src/metadata/orm_profiler/validations/table/table_column_count_to_equal.py b/ingestion/src/metadata/orm_profiler/validations/table/table_column_count_to_equal.py index 0f604354a8d..2acb8bbaed8 100644 --- a/ingestion/src/metadata/orm_profiler/validations/table/table_column_count_to_equal.py +++ b/ingestion/src/metadata/orm_profiler/validations/table/table_column_count_to_equal.py @@ -49,12 +49,10 @@ def table_column_count_to_equal( status = ( TestCaseStatus.Success - if table_profile.columnCount == test_case.value + if table_profile.columnCount == test_case.columnCount else TestCaseStatus.Failed ) - result = ( - f"Found {table_profile.columnCount} columns vs. the expected {test_case.value}" - ) + result = f"Found {table_profile.columnCount} columns vs. the expected {test_case.columnCount}" return TestCaseResult( executionTime=execution_date.timestamp(), testCaseStatus=status, result=result diff --git a/ingestion/tests/unit/profiler/test_validation_parsing.py b/ingestion/tests/unit/profiler/test_validation_parsing.py new file mode 100644 index 00000000000..827dc5f92c5 --- /dev/null +++ b/ingestion/tests/unit/profiler/test_validation_parsing.py @@ -0,0 +1,182 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Test that we can properly parse JSON definition +of validations +""" +from metadata.generated.schema.tests.column.columnValuesLengthsToBeBetween import ( + ColumnValueLengthsToBeBetween, +) +from metadata.generated.schema.tests.column.columnValuesMissingCountToBeEqual import ( + ColumnValuesMissingCount, +) +from metadata.generated.schema.tests.column.columnValuesToBeBetween import ( + ColumnValuesToBeBetween, +) +from metadata.generated.schema.tests.column.columnValuesToBeNotInSet import ( + ColumnValuesToBeNotInSet, +) +from metadata.generated.schema.tests.column.columnValuesToBeNotNull import ( + ColumnValuesToBeNotNull, +) +from metadata.generated.schema.tests.column.columnValuesToBeUnique import ( + ColumnValuesToBeUnique, +) +from metadata.generated.schema.tests.column.columnValuesToMatchRegex import ( + ColumnValuesToMatchRegex, +) +from metadata.generated.schema.tests.columnTest import ColumnTestCase +from metadata.generated.schema.tests.table.tableColumnCountToEqual import ( + TableColumnCountToEqual, +) +from metadata.generated.schema.tests.table.tableRowCountToBeBetween import ( + TableRowCountToBeBetween, +) +from metadata.generated.schema.tests.table.tableRowCountToEqual import ( + TableRowCountToEqual, +) +from metadata.generated.schema.tests.tableTest import TableTestCase + + +def test_column_values_to_be_unique(): + """ + ColumnValuesToBeUnique + """ + obj = {"config": {}, "columnTestType": "columnValuesToBeUnique"} + + test_case = ColumnTestCase.parse_obj(obj) + + assert isinstance(test_case.config, ColumnValuesToBeUnique) + + +def test_column_values_to_be_not_null(): + """ + ColumnValuesToBeNotNull + """ + obj = {"config": {}, "columnTestType": "columnValuesToBeNotNull"} + + test_case = ColumnTestCase.parse_obj(obj) + + # TODO: we should parse this properly + # assert isinstance(test_case.config, ColumnValuesToBeNotNull) + + +def test_column_values_to_be_between(): + """ + ColumnValuesToBeBetween + """ + obj = { + "config": {"minValue": 6, "maxValue": 10}, + "columnTestType": "columnValuesToBeBetween", + } + + test_case = ColumnTestCase.parse_obj(obj) + + assert isinstance(test_case.config, ColumnValuesToBeBetween) + + +def test_column_value_length_to_be_between(): + """ + ColumnValueLengthsToBeBetween + """ + obj = { + "config": {"minLength": 6, "maxLength": 10}, + "columnTestType": "columnValueLengthsToBeBetween", + } + + test_case = ColumnTestCase.parse_obj(obj) + + assert isinstance(test_case.config, ColumnValueLengthsToBeBetween) + + +def test_column_values_not_in_set(): + """ + ColumnValuesToBeNotInSet + """ + obj = { + "config": {"forbiddenValues": ["random"]}, + "columnTestType": "columnValuesToBeNotInSet", + } + + test_case = ColumnTestCase.parse_obj(obj) + + assert isinstance(test_case.config, ColumnValuesToBeNotInSet) + + +def test_column_values_to_match_regex(): + """ + ColumnValuesToMatchRegex + """ + obj = { + "config": {"regex": "%regex%"}, + "columnTestType": "columnValuesToMatchRegex", + } + + test_case = ColumnTestCase.parse_obj(obj) + + assert isinstance(test_case.config, ColumnValuesToMatchRegex) + + +def test_column_values_missing_count_to_be_equal(): + """ + ColumnValuesMissingCount + """ + obj = { + "config": {"missingCountValue": 10, "missingValueMatch": ["N/A"]}, + "columnTestType": "columnValuesMissingCountToBeEqual", + } + + test_case = ColumnTestCase.parse_obj(obj) + + assert isinstance(test_case.config, ColumnValuesMissingCount) + + +def test_table_row_count_to_equal(): + """ + TableRowCountToEqual + """ + obj = { + "config": {"value": 10}, + "tableTestType": "tableRowCountToEqual", + } + + test_case = TableTestCase.parse_obj(obj) + + assert isinstance(test_case.config, TableRowCountToEqual) + + +def test_table_row_count_to_be_between(): + """ + TableRowCountToBeBetween + """ + obj = { + "config": {"minValue": 10, "maxValue": 100}, + "tableTestType": "tableRowCountToBeBetween", + } + + test_case = TableTestCase.parse_obj(obj) + + assert isinstance(test_case.config, TableRowCountToBeBetween) + + +def test_table_column_count_to_equal(): + """ + TableColumnCountToEqual + """ + obj = { + "config": {"columnCount": 10}, + "tableTestType": "tableColumnCountToEqual", + } + + test_case = TableTestCase.parse_obj(obj) + + assert isinstance(test_case.config, TableColumnCountToEqual) diff --git a/ingestion/tests/unit/profiler/test_validations.py b/ingestion/tests/unit/profiler/test_validations.py index 9188acc987c..842224f51e8 100644 --- a/ingestion/tests/unit/profiler/test_validations.py +++ b/ingestion/tests/unit/profiler/test_validations.py @@ -153,7 +153,7 @@ def test_table_column_count_to_equal(): ) res_ok = validate( - TableColumnCountToEqual(value=5), + TableColumnCountToEqual(columnCount=5), table_profile=table_profile, execution_date=EXECUTION_DATE, ) @@ -164,7 +164,7 @@ def test_table_column_count_to_equal(): ) res_ko = validate( - TableColumnCountToEqual(value=20), + TableColumnCountToEqual(columnCount=20), table_profile=table_profile, execution_date=EXECUTION_DATE, ) @@ -180,7 +180,7 @@ def test_table_column_count_to_equal(): ) res_aborted = validate( - TableColumnCountToEqual(value=5), + TableColumnCountToEqual(columnCount=5), table_profile=table_profile_aborted, execution_date=EXECUTION_DATE, ) @@ -379,7 +379,7 @@ def test_column_value_length_to_be_between(): ) res_ok = validate( - ColumnValueLengthsToBeBetween(minValue=2, maxValue=20), + ColumnValueLengthsToBeBetween(minLength=2, maxLength=20), col_profile=col_profile, execution_date=EXECUTION_DATE, ) @@ -390,7 +390,7 @@ def test_column_value_length_to_be_between(): ) res_ko = validate( - ColumnValueLengthsToBeBetween(minValue=10, maxValue=20), + ColumnValueLengthsToBeBetween(minLength=10, maxLength=20), col_profile=col_profile, execution_date=EXECUTION_DATE, ) @@ -404,7 +404,7 @@ def test_column_value_length_to_be_between(): col_profile_aborted = ColumnProfile(minLength=4) res_aborted = validate( - ColumnValueLengthsToBeBetween(minValue=2, maxValue=20), + ColumnValueLengthsToBeBetween(minLength=2, maxLength=20), col_profile=col_profile_aborted, execution_date=EXECUTION_DATE, )