From 358316917f2e15a9eba41f6c7b7683b4fedfed24 Mon Sep 17 00:00:00 2001 From: Teddy Date: Fri, 23 Feb 2024 09:22:23 +0100 Subject: [PATCH] doc: updated test cases documentation (#15325) --- .../ingestion/workflows/data-quality/tests.md | 214 ++++++++++++++---- 1 file changed, 167 insertions(+), 47 deletions(-) diff --git a/openmetadata-docs/content/v1.3.x/connectors/ingestion/workflows/data-quality/tests.md b/openmetadata-docs/content/v1.3.x/connectors/ingestion/workflows/data-quality/tests.md index 69046df96ca..ead90a85593 100644 --- a/openmetadata-docs/content/v1.3.x/connectors/ingestion/workflows/data-quality/tests.md +++ b/openmetadata-docs/content/v1.3.x/connectors/ingestion/workflows/data-quality/tests.md @@ -39,16 +39,20 @@ Validate the total row count in the table is equal to the given value. **YAML Config** ```yaml -testDefinitionName: tableRowCountToEqual -parameterValues: - - name: value - value: 2 +- name: myTestName + description: test description + testDefinitionName: tableRowCountToEqual + parameterValues: + - name: value + value: 2 ``` **JSON Config** ```json { + "name": "myTestName", + "description": "test description", "testDefinitionName": "tableRowCountToEqual", "parameterValues": [ { @@ -79,8 +83,10 @@ Any of those two need to be informed. **YAML Config** ```yaml -testDefinitionName: tableRowCountToBeBetween -parameterValues: +- name: myTestName + description: test description + testDefinitionName: tableRowCountToBeBetween + parameterValues: - name: minValue value: 10 - name: maxValue @@ -91,6 +97,8 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", "testDefinitionName": "tableRowCountToBeBetween", "parameterValues": [ { @@ -122,8 +130,10 @@ Validate that the number of columns in a table is equal to a given value. **YAML Config** ```yaml -testDefinitionName: tableColumnCountToEqual -parameterValues: +- name: myTestName + description: test description + testDefinitionName: tableColumnCountToEqual + parameterValues: - name: columnCount value: 5 ``` @@ -132,6 +142,8 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", "testDefinitionName": "tableColumnCountToEqual", "parameterValues": [ { @@ -160,8 +172,10 @@ Validate the number of columns in a table is between the given value **YAML Config** ```yaml -testDefinitionName: tableColumnCountToBeBetween -parameterValues: +- name: myTestName + description: test description + testDefinitionName: tableColumnCountToBeBetween + parameterValues: - name: minColValue value: 5 - name: maxColValue @@ -172,6 +186,8 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", "testDefinitionName": "tableColumnCountToBeBetween", "parameterValues": [ { @@ -203,8 +219,10 @@ Validate a column name is present in the table **YAML Config** ```yaml -testDefinitionName: tableColumnNameToExist -parameterValues: +- name: myTestName + description: test description + testDefinitionName: tableColumnNameToExist + parameterValues: - name: columnName value: order_id ``` @@ -213,6 +231,7 @@ parameterValues: ```json { + "myTestName": "myTestName", "testDefinitionName": "tableColumnNameToExist", "parameterValues": [ { @@ -243,8 +262,10 @@ Validate a list of table column name matches an expected set of columns **YAML Config** ```yaml -testDefinitionName: tableColumnToMatchSet -parameterValues: +- name: myTestName + description: test description + testDefinitionName: tableColumnToMatchSet + parameterValues: - name: columnNames value: "col1, col2, col3" - name: ordered @@ -255,6 +276,7 @@ parameterValues: ```json { + "myTestName": "myTestName", "testDefinitionName": "tableColumnToMatchSet", "parameterValues": [ { @@ -303,8 +325,10 @@ WHERE lifetime_value < 0; **YAML Config** ```yaml -testDefinitionName: tableCustomSQLQuery -parameterValues: +- name: myTestName + description: test description + testDefinitionName: tableCustomSQLQuery + parameterValues: - name: sqlExpression value: > SELECT @@ -317,6 +341,8 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", "testDefinitionName": "tableCustomSQLQuery", "parameterValues": [ { @@ -354,8 +380,10 @@ The Table Row Inserted Count To Be Between cannot be executed against tables tha **YAML Config** ```yaml -testDefinitionName: tableRowInsertedCountToBeBetween -parameterValues: +- name: myTestName + description: test description + testDefinitionName: tableRowInsertedCountToBeBetween + parameterValues: - name: min value: 10 - name: max @@ -372,6 +400,8 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", "testDefinitionName": "tableRowInsertedCountToBeBetween", "parameterValues": [ { @@ -433,8 +463,11 @@ Makes sure that there are no duplicate values in a given column. **YAML Config** ```yaml -testDefinitionName: columnValuesToBeUnique -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValuesToBeUnique + parameterValues: - name: columnNames value: true ``` @@ -443,6 +476,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValuesToBeUnique", "parameterValues": [ { @@ -470,8 +506,11 @@ Validates that there are no null values in the column. **YAML Config** ```yaml -testDefinitionName: columnValuesToBeNotNull -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValuesToBeNotNull + parameterValues: - name: columnValuesToBeNotNull value: true ``` @@ -480,6 +519,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValuesToBeNotNull", "parameterValues": [ { @@ -517,8 +559,11 @@ The other databases will fall back to the `LIKE` expression **YAML Config** ```yaml -testDefinitionName: columnValuesToMatchRegex -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValuesToMatchRegex + parameterValues: - name: regex value: "%something%" ``` @@ -527,6 +572,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValuesToMatchRegex", "parameterValues": [ { @@ -564,8 +612,11 @@ The other databases will fall back to the `LIKE` expression **YAML Config** ```yaml -testDefinitionName: columnValuesToMatchRegex -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValuesToMatchRegex + parameterValues: - name: forbiddenRegex value: "%something%" ``` @@ -574,6 +625,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValuesToMatchRegex", "parameterValues": [ { @@ -611,6 +665,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValuesToBeInSet", "parameterValues": [ { @@ -641,8 +698,11 @@ Validate that there are no values in a column in a set of forbidden values. **YAML Config** ```yaml -testDefinitionName: columnValuesToBeNotInSet -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValuesToBeNotInSet + parameterValues: - name: forbiddenValues value: ["forbidden1", "forbidden2"] ``` @@ -651,6 +711,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValuesToBeNotInSet", "parameterValues": [ { @@ -689,8 +752,11 @@ Any of those two need to be informed. **YAML Config** ```yaml -testDefinitionName: columnValuesToBeBetween -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValuesToBeBetween + parameterValues: - name: minValue value: ["forbidden1", "forbidden2"] ``` @@ -699,6 +765,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValuesToBeBetween", "parameterValues": [ { @@ -730,8 +799,11 @@ Validates that the number of missing values matches a given number. Missing valu **YAML Config** ```yaml -testDefinitionName: columnValuesMissingCountToBeEqual -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValuesMissingCountToBeEqual + parameterValues: - name: missingValueMatch value: ["NA", "N/A"] - name: missingCountValue @@ -742,6 +814,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValuesMissingCountToBeEqual", "parameterValues": [ { @@ -763,6 +838,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValuesMissingCountToBeEqual", "parameterValues": [ { @@ -805,8 +883,11 @@ Any of those two need to be informed. **YAML Config** ```yaml -testDefinitionName: columnValueLengthsToBeBetween -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValueLengthsToBeBetween + parameterValues: - name: minLength value: 50 - name: maxLength @@ -817,6 +898,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValueLengthsToBeBetween", "parameterValues": [ { @@ -854,8 +938,11 @@ Validate the maximum value of a column is between a specific range **YAML Config** ```yaml -testDefinitionName: columnValueMaxToBeBetween -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValueMaxToBeBetween + parameterValues: - name: minValueForMaxInCol value: 50 - name: maxValueForMaxInCol @@ -866,6 +953,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValueMaxToBeBetween", "parameterValues": [ { @@ -903,8 +993,11 @@ Validate the minimum value of a column is between a specific range **YAML Config** ```yaml -testDefinitionName: columnValueMinToBeBetween -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValueMinToBeBetween + parameterValues: - name: minValueForMinInCol value: 10 - name: maxValueForMinInCol @@ -914,7 +1007,10 @@ parameterValues: **JSON Config** ```json -{ +{ + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValueMinToBeBetween", "parameterValues": [ { @@ -952,8 +1048,11 @@ Validate the mean of a column is between a specific range **YAML Config** ```yaml -testDefinitionName: columnValueMeanToBeBetween -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValueMeanToBeBetween + parameterValues: - name: minValueForMeanInCol value: 5 - name: maxValueForMeanInCol @@ -964,6 +1063,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValueMeanToBeBetween", "parameterValues": [ { @@ -1001,8 +1103,11 @@ Validate the median of a column is between a specific range **YAML Config** ```yaml -testDefinitionName: columnValueMedianToBeBetween -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValueMedianToBeBetween + parameterValues: - name: minValueForMedianInCol value: 5 - name: maxValueForMedianInCol @@ -1013,6 +1118,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValueMedianToBeBetween", "parameterValues": [ { @@ -1050,8 +1158,11 @@ Validate the sum of a column is between a specific range **YAML Config** ```yaml -testDefinitionName: columnValueMedianToBeBetween -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValueMedianToBeBetween + parameterValues: - name: minValueForMedianInCol value: 5 - name: maxValueForMedianInCol @@ -1062,6 +1173,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValueMedianToBeBetween", "parameterValues": [ { @@ -1099,8 +1213,11 @@ Validate the standard deviation of a column is between a specific range **YAML Config** ```yaml -testDefinitionName: columnValueStdDevToBeBetween -parameterValues: +- name: myTestName + description: test description + columnName: columnName + testDefinitionName: columnValueStdDevToBeBetween + parameterValues: - name: minValueForStdDevInCol value: 5 - name: maxValueForStdDevInCol @@ -1111,6 +1228,9 @@ parameterValues: ```json { + "name": "myTestName", + "description": "test description", + "columnName": "columnName", "testDefinitionName": "columnValueStdDevToBeBetween", "parameterValues": [ {