yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-31 01:15:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

ISSUE #24020: add supportedServcices for relavnt service DQ display (#24706)

Browse Source 
* fix: add supportedServcices for relavnt service DQ display

data diff is not supported by all services. We need to only
display it on supported services

* fix: added query param and create filed
This commit is contained in:
Teddy 2025-12-05 15:40:33 +01:00 committed by GitHub
parent d419bedd60
commit d5c6e5b19a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 154 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
bootstrap/sql/migrations/native/1.11.1/mysql/postDataMigrationSQLScript.sql Normal file
View File

@ -0,0 +1,11 @@
UPDATE test_definition
SET json = JSON_SET(
json,
'$.supportedServices',
JSON_ARRAY('Snowflake', 'BigQuery', 'Athena', 'Redshift', 'Postgres', 'MySQL', 'Mssql', 'Oracle', 'Trino', 'SapHana')
)
WHERE name = 'tableDiff'
AND (
JSON_EXTRACT(json, '$.supportedServices') IS NULL
OR JSON_LENGTH(JSON_EXTRACT(json, '$.supportedServices')) = 0
);

11
bootstrap/sql/migrations/native/1.11.1/postgres/postDataMigrationSQLScript.sql Normal file
View File

@ -0,0 +1,11 @@
UPDATE test_definition
SET json = jsonb_set(
json::jsonb,
'{supportedServices}',
'["Snowflake", "BigQuery", "Athena", "Redshift", "Postgres", "MySQL", "Mssql", "Oracle", "Trino", "SapHana"]'::jsonb
)
WHERE name = 'tableDiff'
AND (
json->'supportedServices' IS NULL
OR json->'supportedServices' = '[]'::jsonb
);

54
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -6229,9 +6229,13 @@ public interface CollectionDAO {
String entityType = filter.getQueryParam("entityType");
String testPlatform = filter.getQueryParam("testPlatform");
String supportedDataType = filter.getQueryParam("supportedDataType");
String supportedService = filter.getQueryParam("supportedService");
String condition = filter.getCondition();
if (entityType == null && testPlatform == null && supportedDataType == null) {
if (entityType == null
&& testPlatform == null
&& supportedDataType == null
&& supportedService == null) {
return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId);
}
@ -6259,6 +6263,18 @@ public interface CollectionDAO {
psqlCondition.append("AND json->>'supportedDataTypes' LIKE :supportedDataTypeLike ");
}
if (supportedService != null) {
filter.queryParams.put("supportedServiceLike", String.format("%%%s%%", supportedService));
mysqlCondition.append(
"AND (json_extract(json, '$.supportedServices') = JSON_ARRAY() "
+ "OR json_extract(json, '$.supportedServices') IS NULL "
+ "OR json_extract(json, '$.supportedServices') LIKE :supportedServiceLike) ");
psqlCondition.append(
"AND (json->>'supportedServices' = '[]' "
+ "OR json->>'supportedServices' IS NULL "
+ "OR json->>'supportedServices' LIKE :supportedServiceLike) ");
}
return listBefore(
getTableName(),
filter.getQueryParams(),
@ -6274,9 +6290,13 @@ public interface CollectionDAO {
String entityType = filter.getQueryParam("entityType");
String testPlatform = filter.getQueryParam("testPlatform");
String supportedDataType = filter.getQueryParam("supportedDataType");
String supportedService = filter.getQueryParam("supportedService");
String condition = filter.getCondition();
if (entityType == null && testPlatform == null && supportedDataType == null) {
if (entityType == null
&& testPlatform == null
&& supportedDataType == null
&& supportedService == null) {
return EntityDAO.super.listAfter(filter, limit, afterName, afterId);
}
@ -6304,6 +6324,18 @@ public interface CollectionDAO {
psqlCondition.append("AND json->>'supportedDataTypes' LIKE :supportedDataTypeLike ");
}
if (supportedService != null) {
filter.queryParams.put("supportedServiceLike", String.format("%%%s%%", supportedService));
mysqlCondition.append(
"AND (json_extract(json, '$.supportedServices') = JSON_ARRAY() "
+ "OR json_extract(json, '$.supportedServices') IS NULL "
+ "OR json_extract(json, '$.supportedServices') LIKE :supportedServiceLike) ");
psqlCondition.append(
"AND (json->>'supportedServices' = '[]' "
+ "OR json->>'supportedServices' IS NULL "
+ "OR json->>'supportedServices' LIKE :supportedServiceLike) ");
}
return listAfter(
getTableName(),
filter.getQueryParams(),
@ -6319,9 +6351,13 @@ public interface CollectionDAO {
String entityType = filter.getQueryParam("entityType");
String testPlatform = filter.getQueryParam("testPlatform");
String supportedDataType = filter.getQueryParam("supportedDataType");
String supportedService = filter.getQueryParam("supportedService");
String condition = filter.getCondition();
if (entityType == null && testPlatform == null && supportedDataType == null) {
if (entityType == null
&& testPlatform == null
&& supportedDataType == null
&& supportedService == null) {
return EntityDAO.super.listCount(filter);
}
@ -6348,6 +6384,18 @@ public interface CollectionDAO {
"AND json_extract(json, '$.supportedDataTypes') LIKE :supportedDataTypeLike ");
psqlCondition.append("AND json->>'supportedDataTypes' LIKE :supportedDataTypeLike ");
}
if (supportedService != null) {
filter.queryParams.put("supportedServiceLike", String.format("%%%s%%", supportedService));
mysqlCondition.append(
"AND (json_extract(json, '$.supportedServices') = JSON_ARRAY() "
+ "OR json_extract(json, '$.supportedServices') IS NULL "
+ "OR json_extract(json, '$.supportedServices') LIKE :supportedServiceLike) ");
psqlCondition.append(
"AND (json->>'supportedServices' = '[]' "
+ "OR json->>'supportedServices' IS NULL "
+ "OR json->>'supportedServices' LIKE :supportedServiceLike) ");
}
return listCount(
getTableName(),
filter.getQueryParams(),

1
openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionMapper.java
View File

@ -12,6 +12,7 @@ public class TestDefinitionMapper implements EntityMapper<TestDefinition, Create
.withEntityType(create.getEntityType())
.withTestPlatforms(create.getTestPlatforms())
.withSupportedDataTypes(create.getSupportedDataTypes())
.withSupportedServices(create.getSupportedServices())
.withDisplayName(create.getDisplayName())
.withParameterDefinition(create.getParameterDefinition())
.withName(create.getName());

11
openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java
View File

@ -149,7 +149,13 @@ public class TestDefinitionResource
description = "Filter tests definition by supported data type",
schema = @Schema(implementation = ColumnDataType.class))
@QueryParam("supportedDataType")
String supportedDataTypeParam) {
String supportedDataTypeParam,
@Parameter(
description =
"Filter test definitions by supported service. Returns test definitions that either "
+ "have an empty supportedServices list (supporting all services) or include the specified service.")
@QueryParam("supportedService")
String supportedServiceParam) {
ListFilter filter = new ListFilter(include);
if (entityType != null) {
filter.addQueryParam("entityType", entityType);
@ -160,6 +166,9 @@ public class TestDefinitionResource
if (supportedDataTypeParam != null) {
filter.addQueryParam("supportedDataType", supportedDataTypeParam);
}
if (supportedServiceParam != null) {
filter.addQueryParam("supportedService", supportedServiceParam);
}
return super.listInternal(
uriInfo, securityContext, fieldsParam, filter, limitParam, before, after);
}

14
openmetadata-service/src/main/resources/json/data/tests/tableDiff.json
View File

@ -59,6 +59,18 @@
}
],
"provider": "system",
"dataQualityDimension": "Consistency"
"dataQualityDimension": "Consistency",
"supportedServices": [
"Snowflake",
"BigQuery",
"Athena",
"Redshift",
"Postgres",
"MySQL",
"Mssql",
"Oracle",
"Trino",
"SapHana"
]
}

41
openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestDefinitionResourceTest.java
View File

@ -68,6 +68,45 @@ public class TestDefinitionResourceTest
Assertions.assertTrue(b);
}
@Test
void list_testDefinitionsForSupportedService(TestInfo test) throws HttpResponseException {
CreateTestDefinition createBigQuery =
createRequest("testForBigQuery").withSupportedServices(List.of("BigQuery", "Databricks"));
createEntity(createBigQuery, ADMIN_AUTH_HEADERS);
CreateTestDefinition createSnowflake =
createRequest("testForSnowflake").withSupportedServices(List.of("Snowflake"));
createEntity(createSnowflake, ADMIN_AUTH_HEADERS);
CreateTestDefinition createAllServices =
createRequest("testForAllServices").withSupportedServices(List.of());
createEntity(createAllServices, ADMIN_AUTH_HEADERS);
Map<String, String> params = Map.of("supportedService", "BigQuery", "limit", "1000");
ResultList<TestDefinition> testDefinitions = listEntities(params, ADMIN_AUTH_HEADERS);
boolean allMatch =
testDefinitions.getData().stream()
.allMatch(
t ->
t.getSupportedServices() == null
|| t.getSupportedServices().isEmpty()
|| t.getSupportedServices().contains("BigQuery"));
Assertions.assertTrue(allMatch);
boolean containsBigQueryTest =
testDefinitions.getData().stream().anyMatch(t -> t.getName().equals("testForBigQuery"));
Assertions.assertTrue(containsBigQueryTest);
boolean containsAllServicesTest =
testDefinitions.getData().stream().anyMatch(t -> t.getName().equals("testForAllServices"));
Assertions.assertTrue(containsAllServicesTest);
boolean doesNotContainSnowflakeTest =
testDefinitions.getData().stream().noneMatch(t -> t.getName().equals("testForSnowflake"));
Assertions.assertTrue(doesNotContainSnowflakeTest);
}
@Test
void post_testDefinitionWithoutRequiredFields_4xx(TestInfo test) {
// Test Platform is required field
@ -149,6 +188,7 @@ public class TestDefinitionResourceTest
assertEquals(request.getName(), createdEntity.getName());
assertEquals(request.getDescription(), createdEntity.getDescription());
assertEquals(request.getTestPlatforms(), createdEntity.getTestPlatforms());
assertEquals(request.getSupportedServices(), createdEntity.getSupportedServices());
}
@Override
@ -157,6 +197,7 @@ public class TestDefinitionResourceTest
assertEquals(expected.getName(), updated.getName());
assertEquals(expected.getDescription(), updated.getDescription());
assertEquals(expected.getTestPlatforms(), updated.getTestPlatforms());
assertEquals(expected.getSupportedServices(), updated.getSupportedServices());
}
@Override

8
openmetadata-spec/src/main/resources/json/schema/api/tests/createTestDefinition.json
View File

@ -47,6 +47,14 @@
"$ref": "../../tests/testDefinition.json#/definitions/testCaseParameterDefinition"
}
},
"supportedServices": {
"description": "List of services that this test definition supports. When empty, it implies all services are supported.",
"type": "array",
"items": {
"type": "string"
},
"default": []
},
"domains" : {
"description": "Fully qualified names of the domains the Test Definition belongs to.",
"type": "array",

8
openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json
View File

@ -215,6 +215,14 @@
"description": "When `true` indicates the test case supports dynamic assertions.",
"type": "boolean",
"default": false
},
"supportedServices": {
"description": "List of services that this test definition supports. When empty, it implies all services are supported.",
"type": "array",
"items": {
"type": "string"
},
"default": []
}
},
"required": ["name", "description", "testPlatforms"],