diff --git a/bootstrap/sql/migrations/native/1.11.1/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.11.1/mysql/postDataMigrationSQLScript.sql new file mode 100644 index 00000000000..1bbc5192e62 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.11.1/mysql/postDataMigrationSQLScript.sql @@ -0,0 +1,11 @@ +UPDATE test_definition +SET json = JSON_SET( + json, + '$.supportedServices', + JSON_ARRAY('Snowflake', 'BigQuery', 'Athena', 'Redshift', 'Postgres', 'MySQL', 'Mssql', 'Oracle', 'Trino', 'SapHana') +) +WHERE name = 'tableDiff' + AND ( + JSON_EXTRACT(json, '$.supportedServices') IS NULL + OR JSON_LENGTH(JSON_EXTRACT(json, '$.supportedServices')) = 0 + ); \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.11.1/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.11.1/postgres/postDataMigrationSQLScript.sql new file mode 100644 index 00000000000..a4ed078761f --- /dev/null +++ b/bootstrap/sql/migrations/native/1.11.1/postgres/postDataMigrationSQLScript.sql @@ -0,0 +1,11 @@ +UPDATE test_definition +SET json = jsonb_set( + json::jsonb, + '{supportedServices}', + '["Snowflake", "BigQuery", "Athena", "Redshift", "Postgres", "MySQL", "Mssql", "Oracle", "Trino", "SapHana"]'::jsonb +) +WHERE name = 'tableDiff' + AND ( + json->'supportedServices' IS NULL + OR json->'supportedServices' = '[]'::jsonb + ); \ No newline at end of file diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index afe4164bb07..87e500e9c9c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -6229,9 +6229,13 @@ public interface CollectionDAO { String entityType = filter.getQueryParam("entityType"); String testPlatform = filter.getQueryParam("testPlatform"); String supportedDataType = filter.getQueryParam("supportedDataType"); + String supportedService = filter.getQueryParam("supportedService"); String condition = filter.getCondition(); - if (entityType == null && testPlatform == null && supportedDataType == null) { + if (entityType == null + && testPlatform == null + && supportedDataType == null + && supportedService == null) { return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); } @@ -6259,6 +6263,18 @@ public interface CollectionDAO { psqlCondition.append("AND json->>'supportedDataTypes' LIKE :supportedDataTypeLike "); } + if (supportedService != null) { + filter.queryParams.put("supportedServiceLike", String.format("%%%s%%", supportedService)); + mysqlCondition.append( + "AND (json_extract(json, '$.supportedServices') = JSON_ARRAY() " + + "OR json_extract(json, '$.supportedServices') IS NULL " + + "OR json_extract(json, '$.supportedServices') LIKE :supportedServiceLike) "); + psqlCondition.append( + "AND (json->>'supportedServices' = '[]' " + + "OR json->>'supportedServices' IS NULL " + + "OR json->>'supportedServices' LIKE :supportedServiceLike) "); + } + return listBefore( getTableName(), filter.getQueryParams(), @@ -6274,9 +6290,13 @@ public interface CollectionDAO { String entityType = filter.getQueryParam("entityType"); String testPlatform = filter.getQueryParam("testPlatform"); String supportedDataType = filter.getQueryParam("supportedDataType"); + String supportedService = filter.getQueryParam("supportedService"); String condition = filter.getCondition(); - if (entityType == null && testPlatform == null && supportedDataType == null) { + if (entityType == null + && testPlatform == null + && supportedDataType == null + && supportedService == null) { return EntityDAO.super.listAfter(filter, limit, afterName, afterId); } @@ -6304,6 +6324,18 @@ public interface CollectionDAO { psqlCondition.append("AND json->>'supportedDataTypes' LIKE :supportedDataTypeLike "); } + if (supportedService != null) { + filter.queryParams.put("supportedServiceLike", String.format("%%%s%%", supportedService)); + mysqlCondition.append( + "AND (json_extract(json, '$.supportedServices') = JSON_ARRAY() " + + "OR json_extract(json, '$.supportedServices') IS NULL " + + "OR json_extract(json, '$.supportedServices') LIKE :supportedServiceLike) "); + psqlCondition.append( + "AND (json->>'supportedServices' = '[]' " + + "OR json->>'supportedServices' IS NULL " + + "OR json->>'supportedServices' LIKE :supportedServiceLike) "); + } + return listAfter( getTableName(), filter.getQueryParams(), @@ -6319,9 +6351,13 @@ public interface CollectionDAO { String entityType = filter.getQueryParam("entityType"); String testPlatform = filter.getQueryParam("testPlatform"); String supportedDataType = filter.getQueryParam("supportedDataType"); + String supportedService = filter.getQueryParam("supportedService"); String condition = filter.getCondition(); - if (entityType == null && testPlatform == null && supportedDataType == null) { + if (entityType == null + && testPlatform == null + && supportedDataType == null + && supportedService == null) { return EntityDAO.super.listCount(filter); } @@ -6348,6 +6384,18 @@ public interface CollectionDAO { "AND json_extract(json, '$.supportedDataTypes') LIKE :supportedDataTypeLike "); psqlCondition.append("AND json->>'supportedDataTypes' LIKE :supportedDataTypeLike "); } + + if (supportedService != null) { + filter.queryParams.put("supportedServiceLike", String.format("%%%s%%", supportedService)); + mysqlCondition.append( + "AND (json_extract(json, '$.supportedServices') = JSON_ARRAY() " + + "OR json_extract(json, '$.supportedServices') IS NULL " + + "OR json_extract(json, '$.supportedServices') LIKE :supportedServiceLike) "); + psqlCondition.append( + "AND (json->>'supportedServices' = '[]' " + + "OR json->>'supportedServices' IS NULL " + + "OR json->>'supportedServices' LIKE :supportedServiceLike) "); + } return listCount( getTableName(), filter.getQueryParams(), diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionMapper.java index 1b587fd7747..88e461d5de8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionMapper.java @@ -12,6 +12,7 @@ public class TestDefinitionMapper implements EntityMapper params = Map.of("supportedService", "BigQuery", "limit", "1000"); + ResultList testDefinitions = listEntities(params, ADMIN_AUTH_HEADERS); + + boolean allMatch = + testDefinitions.getData().stream() + .allMatch( + t -> + t.getSupportedServices() == null + || t.getSupportedServices().isEmpty() + || t.getSupportedServices().contains("BigQuery")); + Assertions.assertTrue(allMatch); + + boolean containsBigQueryTest = + testDefinitions.getData().stream().anyMatch(t -> t.getName().equals("testForBigQuery")); + Assertions.assertTrue(containsBigQueryTest); + + boolean containsAllServicesTest = + testDefinitions.getData().stream().anyMatch(t -> t.getName().equals("testForAllServices")); + Assertions.assertTrue(containsAllServicesTest); + + boolean doesNotContainSnowflakeTest = + testDefinitions.getData().stream().noneMatch(t -> t.getName().equals("testForSnowflake")); + Assertions.assertTrue(doesNotContainSnowflakeTest); + } + @Test void post_testDefinitionWithoutRequiredFields_4xx(TestInfo test) { // Test Platform is required field @@ -149,6 +188,7 @@ public class TestDefinitionResourceTest assertEquals(request.getName(), createdEntity.getName()); assertEquals(request.getDescription(), createdEntity.getDescription()); assertEquals(request.getTestPlatforms(), createdEntity.getTestPlatforms()); + assertEquals(request.getSupportedServices(), createdEntity.getSupportedServices()); } @Override @@ -157,6 +197,7 @@ public class TestDefinitionResourceTest assertEquals(expected.getName(), updated.getName()); assertEquals(expected.getDescription(), updated.getDescription()); assertEquals(expected.getTestPlatforms(), updated.getTestPlatforms()); + assertEquals(expected.getSupportedServices(), updated.getSupportedServices()); } @Override diff --git a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestDefinition.json b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestDefinition.json index 50425c02df9..ca2f6a5cb15 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestDefinition.json @@ -47,6 +47,14 @@ "$ref": "../../tests/testDefinition.json#/definitions/testCaseParameterDefinition" } }, + "supportedServices": { + "description": "List of services that this test definition supports. When empty, it implies all services are supported.", + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, "domains" : { "description": "Fully qualified names of the domains the Test Definition belongs to.", "type": "array", diff --git a/openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json b/openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json index f7f3c84fd15..befa4c6e0bc 100644 --- a/openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json @@ -215,6 +215,14 @@ "description": "When `true` indicates the test case supports dynamic assertions.", "type": "boolean", "default": false + }, + "supportedServices": { + "description": "List of services that this test definition supports. When empty, it implies all services are supported.", + "type": "array", + "items": { + "type": "string" + }, + "default": [] } }, "required": ["name", "description", "testPlatforms"],