From 37a7cf99b66fd39aaedbf520d6c5d82837936477 Mon Sep 17 00:00:00 2001 From: Teddy Date: Wed, 28 Jun 2023 18:02:49 +0200 Subject: [PATCH] fix: test defintion migration for parameter definition changes (#12207) --- .../v013__create_db_connection_info.sql | 198 +++++++++++++++++- .../v013__create_db_connection_info.sql | 67 +++++- .../json/data/tests/tableCustomSQLQuery.json | 2 +- 3 files changed, 264 insertions(+), 3 deletions(-) diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v013__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v013__create_db_connection_info.sql index a0142f68fb1..1409de95656 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v013__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v013__create_db_connection_info.sql @@ -148,4 +148,200 @@ CREATE TABLE IF NOT EXISTS SERVER_MIGRATION_SQL_LOGS ( sqlStatement VARCHAR(10000) NOT NULL, checksum VARCHAR(256) PRIMARY KEY, executedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); \ No newline at end of file +); + +-- Update test definition parameterValues +UPDATE test_definition +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.parameterDefinition'), + '$.parameterDefinition', + JSON_ARRAY( + JSON_OBJECT( + 'name', 'minValueForMeanInCol', + 'dataType', 'INT', + 'required', false, + 'description', 'Expected mean value for the column to be greater or equal than', + 'displayName', 'Min' + ), + JSON_OBJECT( + 'name', 'maxValueForMeanInCol', + 'dataType', 'INT', + 'required', false, + 'description', 'Expected mean value for the column to be lower or equal than', + 'displayName', 'Max' + ) + ) +) +WHERE name = 'columnValueMeanToBeBetween'; + +UPDATE test_definition +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.parameterDefinition'), + '$.parameterDefinition', + JSON_ARRAY( + JSON_OBJECT( + 'name', 'minValueForMedianInCol', + 'dataType', 'INT', + 'required', false, + 'description', 'Expected median value for the column to be greater or equal than', + 'displayName', 'Min' + ), + JSON_OBJECT( + 'name', 'maxValueForMedianInCol', + 'dataType', 'INT', + 'required', false, + 'description', 'Expected median value for the column to be lower or equal than', + 'displayName', 'Max' + ) + ) +) +WHERE name = 'columnValueMedianToBeBetween'; + +UPDATE test_definition +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.parameterDefinition'), + '$.parameterDefinition', + JSON_ARRAY( + JSON_OBJECT( + 'name', 'minValueForStdDevInCol', + 'dataType', 'INT', + 'required', false, + 'description', 'Expected std. dev value for the column to be greater or equal than', + 'displayName', 'Min' + ), + JSON_OBJECT( + 'name', 'maxValueForStdDevInCol', + 'dataType', 'INT', + 'required', false, + 'description', 'Expected std. dev value for the column to be lower or equal than', + 'displayName', 'Max' + ) + ) +) +WHERE name = 'columnValueStdDevToBeBetween'; + +UPDATE test_definition +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.parameterDefinition'), + '$.parameterDefinition', + JSON_ARRAY( + JSON_OBJECT( + 'name', 'minLength', + 'dataType', 'INT', + 'required', false, + 'description', 'The {minLength} for the column value. If minLength is not included, maxLength is treated as upperBound and there will be no minimum value length', + 'displayName', 'Min' + ), + JSON_OBJECT( + 'name', 'maxLength', + 'dataType', 'INT', + 'required', false, + 'description', 'The {maxLength} for the column value. if maxLength is not included, minLength is treated as lowerBound and there will be no maximum value length', + 'displayName', 'Max' + ) + ) +) +WHERE name = 'columnValueLengthsToBeBetween'; + +UPDATE test_definition +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.parameterDefinition'), + '$.parameterDefinition', + JSON_ARRAY( + JSON_OBJECT( + 'name', 'minValue', + 'dataType', 'INT', + 'required', false, + 'description', 'The {minValue} value for the column entry. If minValue is not included, maxValue is treated as upperBound and there will be no minimum', + 'displayName', 'Min' + ), + JSON_OBJECT( + 'name', 'maxValue', + 'dataType', 'INT', + 'required', false, + 'description', 'The {maxValue} value for the column entry. if maxValue is not included, minValue is treated as lowerBound and there will be no maximum', + 'displayName', 'Max' + ) + ) +) +WHERE name = 'columnValuesToBeBetween'; + +UPDATE test_definition +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.parameterDefinition'), + '$.parameterDefinition', + JSON_ARRAY( + JSON_OBJECT( + 'name', 'columnNames', + 'dataType', 'STRING', + 'required', true, + 'description', 'Expected columns names of the table to match the ones in {Column Names} -- should be a coma separated string', + 'displayName', 'Column Names' + ), + JSON_OBJECT( + 'name', 'ordered', + 'dataType', 'BOOLEAN', + 'required', false, + 'description', 'Whether or not to considered the order of the list when performing the match check', + 'displayName', 'Ordered' + ) + ) +) +WHERE name = 'tableColumnToMatchSet'; + +UPDATE test_definition +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.parameterDefinition'), + '$.parameterDefinition', + JSON_ARRAY( + JSON_OBJECT( + 'name', 'minValue', + 'dataType', 'INT', + 'required', false, + 'description', 'Expected number of columns should be greater than or equal to {minValue}. If minValue is not included, maxValue is treated as upperBound and there will be no minimum', + 'displayName', 'Min' + ), + JSON_OBJECT( + 'name', 'maxValue', + 'dataType', 'INT', + 'required', false, + 'description', 'Expected number of columns should be less than or equal to {maxValue}. If maxValue is not included, minValue is treated as lowerBound and there will be no maximum', + 'displayName', 'Max' + ) + ) +) +WHERE name = 'tableRowCountToBeBetween'; + +UPDATE test_definition +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.parameterDefinition'), + '$.parameterDefinition', + JSON_ARRAY( + JSON_OBJECT( + 'name', 'sqlExpression', + 'displayName', 'SQL Expression', + 'description', 'SQL expression to run against the table', + 'dataType', 'STRING', + 'required', 'true' + ), + JSON_OBJECT( + 'name', 'strategy', + 'displayName', 'Strategy', + 'description', 'Strategy to use to run the custom SQL query (i.e. `SELECT COUNT()` or `SELECT (defaults to ROWS)', + 'dataType', 'ARRAY', + 'optionValues', JSON_ARRAY( + 'ROWS', + 'COUNT' + ), + 'required', false + ), + JSON_OBJECT( + 'name', 'threshold', + 'displayName', 'Threshold', + 'description', 'Threshold to use to determine if the test passes or fails (defaults to 0).', + 'dataType', 'NUMBER', + 'required', false + ) + ) +) +WHERE name = 'tableCustomSQLQuery'; \ No newline at end of file diff --git a/bootstrap/sql/org.postgresql.Driver/v013__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v013__create_db_connection_info.sql index e7c5b3a3698..ce43694f1ae 100644 --- a/bootstrap/sql/org.postgresql.Driver/v013__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v013__create_db_connection_info.sql @@ -111,4 +111,69 @@ CREATE TABLE IF NOT EXISTS SERVER_MIGRATION_SQL_LOGS ( sqlStatement VARCHAR(10000) NOT NULL, checksum VARCHAR(256) PRIMARY KEY, executedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); \ No newline at end of file +); + +-- Update test definition parameterValues +update test_definition +set json = jsonb_set( + json, + '{parameterDefinition}', + '[{"name": "minValueForMeanInCol", "dataType": "INT", "required": false, "description": "Expected mean value for the column to be greater or equal than", "displayName": "Min", "optionValues": []}, {"name": "maxValueForMeanInCol", "dataType": "INT", "required": false, "description": "Expected mean value for the column to be lower or equal than", "displayName": "Max", "optionValues": []}]' + ) +where name = 'columnValueMeanToBeBetween'; + +update test_definition +set json = jsonb_set( + json, + '{parameterDefinition}', + '[{"name": "minValueForMedianInCol", "dataType": "INT", "required": false, "description": "Expected median value for the column to be greater or equal than", "displayName": "Min", "optionValues": []}, {"name": "maxValueForMedianInCol", "dataType": "INT", "required": false, "description": "Expected median value for the column to be lower or equal than", "displayName": "Max", "optionValues": []}]' + ) +where name = 'columnValueMedianToBeBetween'; + +update test_definition +set json = jsonb_set( + json, + '{parameterDefinition}', + '[{"name": "minValueForStdDevInCol", "dataType": "INT", "required": false, "description": "Expected std. dev value for the column to be greater or equal than", "displayName": "Min", "optionValues": []}, {"name": "maxValueForStdDevInCol", "dataType": "INT", "required": false, "description": "Expected std. dev value for the column to be lower or equal than", "displayName": "Max", "optionValues": []}]' + ) +where name = 'columnValueStdDevToBeBetween'; + +update test_definition +set json = jsonb_set( + json, + '{parameterDefinition}', + '[{"name": "minLength", "dataType": "INT", "required": false, "description": "The {minLength} for the column value. If minLength is not included, maxLength is treated as upperBound and there will be no minimum value length", "displayName": "Min", "optionValues": []}, {"name": "maxLength", "dataType": "INT", "required": false, "description": "The {maxLength} for the column value. if maxLength is not included, minLength is treated as lowerBound and there will be no maximum value length", "displayName": "Max", "optionValues": []}]' + ) +where name = 'columnValueLengthsToBeBetween'; + +update test_definition +set json = jsonb_set( + json, + '{parameterDefinition}', + '[{"name": "minValue", "dataType": "INT", "required": false, "description": "The {minValue} value for the column entry. If minValue is not included, maxValue is treated as upperBound and there will be no minimum", "displayName": "Min", "optionValues": []}, {"name": "maxValue", "dataType": "INT", "required": false, "description": "The {maxValue} value for the column entry. if maxValue is not included, minValue is treated as lowerBound and there will be no maximum", "displayName": "Max", "optionValues": []}]' + ) +where name = 'columnValuesToBeBetween'; + +update test_definition +set json = jsonb_set( + json, + '{parameterDefinition}', + '[{"name": "columnNames", "dataType": "STRING", "required": true, "description": "Expected columns names of the table to match the ones in {Column Names} -- should be a coma separated string", "displayName": "Column Names", "optionValues": []}, {"name": "ordered", "dataType": "BOOLEAN", "required": false, "description": "Whether or not to considered the order of the list when performing the match check", "displayName": "Ordered", "optionValues": []}]' + ) +where name = 'tableColumnToMatchSet'; + +update test_definition +set json = jsonb_set( + json, + '{parameterDefinition}', + '[{"name": "minValue", "dataType": "INT", "required": false, "description": "Expected number of columns should be greater than or equal to {minValue}. If minValue is not included, maxValue is treated as upperBound and there will be no minimum", "displayName": "Min", "optionValues": []}, {"name": "maxValue", "dataType": "INT", "required": false, "description": "Expected number of columns should be less than or equal to {maxValue}. If maxValue is not included, minValue is treated as lowerBound and there will be no maximum", "displayName": "Max", "optionValues": []}]' + ) +where name = 'tableRowCountToBeBetween'; + +update test_definition +set json = jsonb_set( + json, + '{parameterDefinition}', + '[{"name":"sqlExpression","displayName":"SQL Expression","description":"SQL expression to run against the table","dataType":"STRING","required":"true"},{"name":"strategy","displayName":"Strategy","description":"Strategy to use to run the custom SQL query (i.e. `SELECT COUNT()` or `SELECT (defaults to ROWS)","dataType":"ARRAY","optionValues":["ROWS","COUNT"],"required":false},{"name":"threshold","displayName":"Threshold","description":"Threshold to use to determine if the test passes or fails (defaults to 0).","dataType":"NUMBER","required":false}]' + ) +where name = 'tableCustomSQLQuery'; diff --git a/openmetadata-service/src/main/resources/json/data/tests/tableCustomSQLQuery.json b/openmetadata-service/src/main/resources/json/data/tests/tableCustomSQLQuery.json index fb4124d7e65..d372b50b024 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/tableCustomSQLQuery.json +++ b/openmetadata-service/src/main/resources/json/data/tests/tableCustomSQLQuery.json @@ -16,7 +16,7 @@ { "name": "strategy", "displayName": "Strategy", - "description": "Strategy to use to run the custom SQL query (i.e. `SELECT COUNT()` or `SELECT (defaults to COUNT)", + "description": "Strategy to use to run the custom SQL query (i.e. `SELECT COUNT()` or `SELECT (defaults to ROWS)", "dataType": "ARRAY", "optionValues": ["ROWS", "COUNT"], "required": false