diff --git a/bootstrap/sql/migrations/native/1.4.0/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.4.0/mysql/postDataMigrationSQLScript.sql new file mode 100644 index 00000000000..a3693db8f6a --- /dev/null +++ b/bootstrap/sql/migrations/native/1.4.0/mysql/postDataMigrationSQLScript.sql @@ -0,0 +1,13 @@ +-- Update the relation between testDefinition and testCase to 0 (CONTAINS) +UPDATE entity_relationship +SET relation = 0 +WHERE fromEntity = 'testDefinition' AND toEntity = 'testCase' AND relation != 0; + +-- Update the test definition provider +-- If the test definition has OpenMetadata as a test platform, then the provider is system, else it is user +UPDATE test_definition +SET json = CASE + WHEN JSON_CONTAINS(json, '"OpenMetadata"', '$.testPlatforms') THEN JSON_INSERT(json,'$.provider','system') + ELSE JSON_INSERT(json,'$.provider','user') + END +; diff --git a/bootstrap/sql/migrations/native/1.4.0/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.4.0/postgres/postDataMigrationSQLScript.sql new file mode 100644 index 00000000000..dca6c0b7e3e --- /dev/null +++ b/bootstrap/sql/migrations/native/1.4.0/postgres/postDataMigrationSQLScript.sql @@ -0,0 +1,13 @@ +-- Update the relation between testDefinition and testCase to 0 (CONTAINS) +UPDATE entity_relationship +SET relation = 0 +WHERE fromEntity = 'testDefinition' AND toEntity = 'testCase' AND relation != 0; + +-- Update the test definition provider +-- If the test definition has OpenMetadata as a test platform, then the provider is system, else it is user +UPDATE test_definition +SET json = + case + when json->'testPlatforms' @> '"OpenMetadata"' then jsonb_set(json,'{provider}','"system"',true) + else jsonb_set(json,'{provider}','"user"', true) + end; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java index f9dc9bcaefa..ae0d2fdcaa4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java @@ -193,7 +193,7 @@ public class TestCaseRepository extends EntityRepository { } private EntityReference getTestDefinition(TestCase test) { - return getFromEntityRef(test.getId(), Relationship.APPLIED_TO, TEST_DEFINITION, true); + return getFromEntityRef(test.getId(), Relationship.CONTAINS, TEST_DEFINITION, true); } private void validateTestParameters( @@ -247,7 +247,7 @@ public class TestCaseRepository extends EntityRepository { test.getId(), TEST_DEFINITION, TEST_CASE, - Relationship.APPLIED_TO); + Relationship.CONTAINS); } @Override @@ -900,7 +900,7 @@ public class TestCaseRepository extends EntityRepository { TEST_DEFINITION, original.getTestDefinition(), updated.getTestDefinition(), - Relationship.APPLIED_TO, + Relationship.CONTAINS, TEST_CASE, updated.getId()); recordChange("parameterValues", original.getParameterValues(), updated.getParameterValues()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java index 7c5a8a89091..ece4ab5c9fa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java @@ -389,10 +389,15 @@ public class TestDefinitionResource @QueryParam("hardDelete") @DefaultValue("false") boolean hardDelete, + @Parameter( + description = "Recursively delete this entity and it's children. (Default `false`)") + @QueryParam("recursive") + @DefaultValue("false") + boolean recursive, @Parameter(description = "Id of the test definition", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) { - return delete(uriInfo, securityContext, id, false, hardDelete); + return delete(uriInfo, securityContext, id, recursive, hardDelete); } @DELETE @@ -414,10 +419,15 @@ public class TestDefinitionResource @QueryParam("hardDelete") @DefaultValue("false") boolean hardDelete, + @Parameter( + description = "Recursively delete this entity and it's children. (Default `false`)") + @QueryParam("recursive") + @DefaultValue("false") + boolean recursive, @Parameter(description = "Name of the test definition", schema = @Schema(type = "string")) @PathParam("name") String name) { - return deleteByName(uriInfo, securityContext, name, false, hardDelete); + return deleteByName(uriInfo, securityContext, name, recursive, hardDelete); } @PUT diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValueMaxToBeBetween.json b/openmetadata-service/src/main/resources/json/data/tests/columnValueMaxToBeBetween.json index 633d143cfb7..7f5ab514d6f 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValueMaxToBeBetween.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValueMaxToBeBetween.json @@ -19,6 +19,7 @@ "description": "Expected maximum value in the column to be lower or equal than", "dataType": "INT" } - ] + ], + "provider": "system" } diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValueMeanToBeBetween.json b/openmetadata-service/src/main/resources/json/data/tests/columnValueMeanToBeBetween.json index 5e5fe10a1d6..ec1dc79ad19 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValueMeanToBeBetween.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValueMeanToBeBetween.json @@ -19,6 +19,7 @@ "description": "Expected mean value for the column to be lower or equal than", "dataType": "INT" } - ] + ], + "provider": "system" } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValueMedianToBeBetween.json b/openmetadata-service/src/main/resources/json/data/tests/columnValueMedianToBeBetween.json index fed513992d2..aba61a8df6a 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValueMedianToBeBetween.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValueMedianToBeBetween.json @@ -19,6 +19,7 @@ "description": "Expected median value for the column to be lower or equal than", "dataType": "INT" } - ] + ], + "provider": "system" } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValueMinToBeBetween.json b/openmetadata-service/src/main/resources/json/data/tests/columnValueMinToBeBetween.json index fa323cefc69..66b76d75ab6 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValueMinToBeBetween.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValueMinToBeBetween.json @@ -19,6 +19,7 @@ "description": "Expect minimum value in the column to be lower or equal than", "dataType": "INT" } - ] + ], + "provider": "system" } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValueStdDevToBeBetween.json b/openmetadata-service/src/main/resources/json/data/tests/columnValueStdDevToBeBetween.json index dc9e801e31c..6d242a76455 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValueStdDevToBeBetween.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValueStdDevToBeBetween.json @@ -19,6 +19,7 @@ "description": "Expected std. dev value for the column to be lower or equal than", "dataType": "INT" } - ] + ], + "provider": "system" } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValuesLengthsToBeBetween.json b/openmetadata-service/src/main/resources/json/data/tests/columnValuesLengthsToBeBetween.json index 39d0d26a988..ee1a4dea6fd 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesLengthsToBeBetween.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesLengthsToBeBetween.json @@ -20,5 +20,6 @@ "dataType": "INT" } ], - "supportsRowLevelPassedFailed": true + "supportsRowLevelPassedFailed": true, + "provider": "system" } diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValuesMissingCountToBeEqual.json b/openmetadata-service/src/main/resources/json/data/tests/columnValuesMissingCountToBeEqual.json index cfbcc6f72ff..c9af5df3c55 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesMissingCountToBeEqual.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesMissingCountToBeEqual.json @@ -20,5 +20,6 @@ "description": "By default match all null and empty values to be missing. This field allows us to configure additional strings such as N/A, NULL as missing strings as well.", "dataType": "STRING" } - ] + ], + "provider": "system" } diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValuesSumToBeBetween.json b/openmetadata-service/src/main/resources/json/data/tests/columnValuesSumToBeBetween.json index 2822ea418ce..278b5c0149f 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesSumToBeBetween.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesSumToBeBetween.json @@ -19,6 +19,7 @@ "description": "Expected sum values in the column to be lower or equal than", "dataType": "INT" } - ] + ], + "provider": "system" } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeBetween.json b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeBetween.json index e8b74744b1a..79d880998d0 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeBetween.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeBetween.json @@ -20,5 +20,6 @@ "dataType": "INT" } ], - "supportsRowLevelPassedFailed": true + "supportsRowLevelPassedFailed": true, + "provider": "system" } diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeInSet.json b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeInSet.json index af09aceae59..2a990e502fa 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeInSet.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeInSet.json @@ -15,5 +15,6 @@ "required": true } ], - "supportsRowLevelPassedFailed": true + "supportsRowLevelPassedFailed": true, + "provider": "system" } diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotInSet.json b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotInSet.json index c0eae217ef5..2a09b8a6e00 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotInSet.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotInSet.json @@ -15,5 +15,6 @@ "required": true } ], - "supportsRowLevelPassedFailed": true + "supportsRowLevelPassedFailed": true, + "provider": "system" } diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotNull.json b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotNull.json index 750f50ecfb5..30dd85f3358 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotNull.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotNull.json @@ -6,5 +6,6 @@ "entityType": "COLUMN", "testPlatforms": ["OpenMetadata"], "supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"], - "supportsRowLevelPassedFailed": true + "supportsRowLevelPassedFailed": true, + "provider": "system" } diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeUnique.json b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeUnique.json index 5a890b4e425..33d3f067930 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeUnique.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeUnique.json @@ -6,5 +6,6 @@ "entityType": "COLUMN", "testPlatforms": ["OpenMetadata"], "supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"], - "supportsRowLevelPassedFailed": true + "supportsRowLevelPassedFailed": true, + "provider": "system" } diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToMatchRegex.json b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToMatchRegex.json index 4a9b1f5a7d0..b7a9c2293a0 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToMatchRegex.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToMatchRegex.json @@ -15,5 +15,6 @@ "required": true } ], - "supportsRowLevelPassedFailed": true + "supportsRowLevelPassedFailed": true, + "provider": "system" } diff --git a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToNotMatchRegex.json b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToNotMatchRegex.json index a03b1cd2f10..c9be6ed4efe 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToNotMatchRegex.json +++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToNotMatchRegex.json @@ -15,5 +15,6 @@ "required": true } ], - "supportsRowLevelPassedFailed": true + "supportsRowLevelPassedFailed": true, + "provider": "system" } diff --git a/openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToBeBetween.json b/openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToBeBetween.json index 4de215e2e25..f4b5f0fd8f0 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToBeBetween.json +++ b/openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToBeBetween.json @@ -18,5 +18,6 @@ "description": "Expected number of columns should be less than or equal to {maxValue}. If maxValue is not included, minValue is treated as lowerBound and there will be no maximum number of column", "dataType": "INT" } - ] + ], + "provider": "system" } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToEqual.json b/openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToEqual.json index d4494915282..9b74527806a 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToEqual.json +++ b/openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToEqual.json @@ -13,5 +13,6 @@ "dataType": "INT", "required": true } - ] + ], + "provider": "system" } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/tests/tableColumnNameToExist.json b/openmetadata-service/src/main/resources/json/data/tests/tableColumnNameToExist.json index d1d88edbb71..d2b0b1479fa 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/tableColumnNameToExist.json +++ b/openmetadata-service/src/main/resources/json/data/tests/tableColumnNameToExist.json @@ -13,6 +13,7 @@ "dataType": "STRING", "required": true } - ] + ], + "provider": "system" } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/tests/tableColumnToMatchSet.json b/openmetadata-service/src/main/resources/json/data/tests/tableColumnToMatchSet.json index 87db21db025..c1692bd259a 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/tableColumnToMatchSet.json +++ b/openmetadata-service/src/main/resources/json/data/tests/tableColumnToMatchSet.json @@ -19,6 +19,7 @@ "description": "Whether or not to considered the order of the list when performing the match check", "dataType": "BOOLEAN" } - ] + ], + "provider": "system" } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/tests/tableCustomSQLQuery.json b/openmetadata-service/src/main/resources/json/data/tests/tableCustomSQLQuery.json index d372b50b024..70b8af767f0 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/tableCustomSQLQuery.json +++ b/openmetadata-service/src/main/resources/json/data/tests/tableCustomSQLQuery.json @@ -28,6 +28,7 @@ "dataType": "NUMBER", "required": false } - ] + ], + "provider": "system" } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/tests/tableRowCountToBeBetween.json b/openmetadata-service/src/main/resources/json/data/tests/tableRowCountToBeBetween.json index 5b1aaee9bb4..100f4aa3e1c 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/tableRowCountToBeBetween.json +++ b/openmetadata-service/src/main/resources/json/data/tests/tableRowCountToBeBetween.json @@ -18,5 +18,6 @@ "description": "Expected number of columns should be less than or equal to {maxValue}. If maxValue is not included, minValue is treated as lowerBound and there will be no maximum", "dataType": "INT" } - ] + ], + "provider": "system" } diff --git a/openmetadata-service/src/main/resources/json/data/tests/tableRowCountToEqual.json b/openmetadata-service/src/main/resources/json/data/tests/tableRowCountToEqual.json index 10b2891583b..487ddb8263c 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/tableRowCountToEqual.json +++ b/openmetadata-service/src/main/resources/json/data/tests/tableRowCountToEqual.json @@ -13,5 +13,6 @@ "dataType": "INT", "required": true } - ] + ], + "provider": "system" } diff --git a/openmetadata-service/src/main/resources/json/data/tests/tableRowInsertedCountToBeBetween.json b/openmetadata-service/src/main/resources/json/data/tests/tableRowInsertedCountToBeBetween.json index 8db90f1dfc3..c0bc69d8bce 100644 --- a/openmetadata-service/src/main/resources/json/data/tests/tableRowInsertedCountToBeBetween.json +++ b/openmetadata-service/src/main/resources/json/data/tests/tableRowInsertedCountToBeBetween.json @@ -41,6 +41,7 @@ "dataType": "INT", "required": true } - ] + ], + "provider": "system" } \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestDefinition.json b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestDefinition.json index 3f30944995f..47b7617401b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestDefinition.json @@ -38,6 +38,9 @@ "$ref": "../../entity/data/table.json#/definitions/dataType" } }, + "provider" : { + "$ref": "../../type/basic.json#/definitions/providerType" + }, "parameterDefinition": { "type": "array", "items": { diff --git a/openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json b/openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json index 9ac82c7aa7f..a6a910c8a2a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json @@ -131,6 +131,9 @@ "$ref": "../type/entityReference.json", "default": null }, + "provider" : { + "$ref": "../type/basic.json#/definitions/providerType" + }, "version": { "description": "Metadata version of the entity.", "$ref": "../type/entityHistory.json#/definitions/entityVersion"