[GEN-876] Implement DQ dimension filter for test cases (#17032)

* fix: typo in key for data dimension * fix: table test key typo in data dimension * fix: typo in migration util * fix: add filter test cases by dimensions in search API * style: ran java linting
2025-08-27 18:36:08 +00:00 · 2024-07-16 08:12:39 +02:00 · 2024-07-16 08:12:39 +02:00 · 924a100ab4
commit 924a100ab4
parent d10c129ed3
34 changed files with 66 additions and 37 deletions
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
@ -1068,8 +1068,13 @@ public abstract class EntityRepository<T extends EntityInterface> {
        entityList.add(withHref(uriInfo, entity));
      }
      return new ResultList<>(entityList, offset, limit, total.intValue());
+    } else {
+      SearchClient.SearchResultListMapper results =
+          searchRepository.listWithOffset(
+              searchListFilter, limit, offset, entityType, searchSortFilter, q);
+      total = results.getTotal();
+      return new ResultList<>(entityList, null, limit, total.intValue());
    }
-    throw new IllegalArgumentException("Limit should be greater than 0");
  }

  @Transaction
--- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v150/MigrationUtil.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v150/MigrationUtil.java
@ -64,7 +64,7 @@ public class MigrationUtil {
                        testCaseDefinition.getFullyQualifiedName());
                    return;
                  }
-                  testCaseDefinition.setDatatQualityDimension(dimension);
+                  testCaseDefinition.setDataQualityDimension(dimension);
                  collectionDAO.testDefinitionDAO().update(testCaseDefinition);
                } catch (Exception e) {
                  LOG.warn("Error migrating test case dimension", e);
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java
@ -91,7 +91,7 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor

  static final String FIELDS = "owner,testSuite,testDefinition,testSuites,incidentId,domain,tags";
  static final String SEARCH_FIELDS_EXCLUDE =
-      "testPlatforms,table,database,databaseSchema,service,testSuite";
+      "testPlatforms,table,database,databaseSchema,service,testSuite,dataQualityDimension";

  @Override
  public TestCase addHref(UriInfo uriInfo, TestCase test) {
@ -316,6 +316,12 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
              schema = @Schema(type = "string"))
          @QueryParam("testPlatforms")
          String testPlatforms,
+      @Parameter(
+              description =
+                  "Filter for test case by data quality dimension (e.g. OpenMetadata, dbt, etc.)",
+              schema = @Schema(type = "string"))
+          @QueryParam("dataQualityDimension")
+          String dataQualityDimension,
      @Parameter(
              description =
                  "Parameter used to filter (inclusive) the test cases by the last execution timestamp (in milliseconds). Must be used in conjunction with `endTimestamp`",
@ -392,6 +398,7 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
    searchListFilter.addQueryParam("testCaseStatus", status);
    searchListFilter.addQueryParam("testCaseType", type);
    searchListFilter.addQueryParam("testPlatforms", testPlatforms);
+    searchListFilter.addQueryParam("dataQualityDimension", dataQualityDimension);
    searchListFilter.addQueryParam("q", q);
    searchListFilter.addQueryParam("excludeFields", SEARCH_FIELDS_EXCLUDE);
    searchListFilter.addQueryParam("includeFields", includeFields);
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java
@ -141,6 +141,7 @@ public class SearchListFilter extends Filter<SearchListFilter> {
    String tags = getQueryParam("tags");
    String tier = getQueryParam("tier");
    String serviceName = getQueryParam("serviceName");
+    String dataQualityDimension = getQueryParam("dataQualityDimension");

    if (tags != null) {
      String tagsList =
@ -209,6 +210,11 @@ public class SearchListFilter extends Filter<SearchListFilter> {
          getTimestampFilter("testCaseResult.timestamp", "lte", Long.parseLong(endTimestamp)));
    }

+    if (dataQualityDimension != null) {
+      conditions.add(
+          String.format("{\"term\": {\"dataQualityDimension\": \"%s\"}}", dataQualityDimension));
+    }
+
    return addCondition(conditions);
  }

--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java
@ -4,11 +4,9 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.UUID;
 import lombok.SneakyThrows;
 import org.openmetadata.schema.tests.TestCase;
 import org.openmetadata.schema.tests.TestDefinition;
-import org.openmetadata.schema.tests.TestPlatform;
 import org.openmetadata.schema.tests.TestSuite;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
@ -33,6 +31,9 @@ public record TestCaseIndex(TestCase testCase) implements SearchIndex {
  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
    // Build Index Doc
    List<SearchSuggest> suggest = new ArrayList<>();
+    TestDefinition testDefinition =
+        Entity.getEntity(
+            Entity.TEST_DEFINITION, testCase.getTestDefinition().getId(), "", Include.ALL);
    suggest.add(SearchSuggest.builder().input(testCase.getFullyQualifiedName()).weight(5).build());
    suggest.add(SearchSuggest.builder().input(testCase.getName()).weight(10).build());
    doc.put(
@ -44,18 +45,13 @@ public record TestCaseIndex(TestCase testCase) implements SearchIndex {
    doc.put("entityType", Entity.TEST_CASE);
    doc.put("owner", getEntityWithDisplayName(testCase.getOwner()));
    doc.put("tags", testCase.getTags());
-    doc.put("testPlatforms", getTestDefinitionPlatforms(testCase.getTestDefinition().getId()));
+    doc.put("testPlatforms", testDefinition.getTestPlatforms());
+    doc.put("dataQualityDimension", testDefinition.getDataQualityDimension());
    doc.put("followers", SearchIndexUtils.parseFollowers(testCase.getFollowers()));
    setParentRelationships(doc, testCase);
    return doc;
  }

-  private List<TestPlatform> getTestDefinitionPlatforms(UUID testDefinitionId) {
-    TestDefinition testDefinition =
-        Entity.getEntity(Entity.TEST_DEFINITION, testDefinitionId, "", Include.ALL);
-    return testDefinition.getTestPlatforms();
-  }
-
  private void setParentRelationships(Map<String, Object> doc, TestCase testCase) {
    // denormalize the parent relationships for search
    EntityReference testSuiteEntityReference = testCase.getTestSuite();
--- a/openmetadata-service/src/main/resources/elasticsearch/en/test_case_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/en/test_case_index_mapping.json
@ -169,6 +169,9 @@
      "entityType": {
        "type": "keyword"
      },
+      "dataQualityDimension": {
+        "type": "keyword"
+      },
      "suggest": {
        "type": "completion",
        "contexts": [
--- a/openmetadata-service/src/main/resources/elasticsearch/jp/test_case_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/jp/test_case_index_mapping.json
@ -192,6 +192,9 @@
      "entityType": {
        "type": "keyword"
      },
+      "dataQualityDimension": {
+        "type": "keyword"
+      },
      "suggest": {
        "type": "completion",
        "contexts": [
--- a/openmetadata-service/src/main/resources/elasticsearch/zh/test_case_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/zh/test_case_index_mapping.json
@ -215,6 +215,9 @@
      "entityType": {
        "type": "keyword"
      },
+      "dataQualityDimension": {
+        "type": "keyword"
+      },
      "suggest": {
        "type": "completion",
        "contexts": [
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValueMaxToBeBetween.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValueMaxToBeBetween.json
@ -30,6 +30,6 @@
  ],
  "supportsDynamicAssertion": true,
  "provider": "system",
-  "datatQualityDimension": "Accuracy"
+  "dataQualityDimension": "Accuracy"
 }
  
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValueMeanToBeBetween.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValueMeanToBeBetween.json
@ -30,6 +30,6 @@
  ],
  "supportsDynamicAssertion": true,
  "provider": "system",
-  "datatQualityDimension": "Accuracy"
+  "dataQualityDimension": "Accuracy"
 }
  
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValueMedianToBeBetween.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValueMedianToBeBetween.json
@ -30,6 +30,6 @@
  ],
  "supportsDynamicAssertion": true,
  "provider": "system",
-  "datatQualityDimension": "Accuracy"
+  "dataQualityDimension": "Accuracy"
 }
  
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValueMinToBeBetween.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValueMinToBeBetween.json
@ -30,6 +30,6 @@
    ],
    "supportsDynamicAssertion": true,
    "provider": "system",
-    "datatQualityDimension": "Accuracy"
+    "dataQualityDimension": "Accuracy"
  }
  
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValueStdDevToBeBetween.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValueStdDevToBeBetween.json
@ -30,6 +30,6 @@
    ],
    "supportsDynamicAssertion": true,
    "provider": "system",
-    "datatQualityDimension": "Accuracy"
+    "dataQualityDimension": "Accuracy"
  }
  
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesLengthsToBeBetween.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesLengthsToBeBetween.json
@ -31,5 +31,5 @@
  "supportsRowLevelPassedFailed": true,
  "supportsDynamicAssertion": true,
  "provider": "system",
-  "datatQualityDimension": "Accuracy"
+  "dataQualityDimension": "Accuracy"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesMissingCountToBeEqual.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesMissingCountToBeEqual.json
@ -22,5 +22,5 @@
    }
  ],
  "provider": "system",
-  "datatQualityDimension": "Completeness"
+  "dataQualityDimension": "Completeness"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesSumToBeBetween.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesSumToBeBetween.json
@ -30,6 +30,6 @@
    ],
    "supportsDynamicAssertion": true,
    "provider": "system",
-    "datatQualityDimension": "Accuracy"
+    "dataQualityDimension": "Accuracy"
  }
  
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeBetween.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeBetween.json
@ -31,5 +31,5 @@
  "supportsRowLevelPassedFailed": true,
  "supportsDynamicAssertion": true,
  "provider": "system",
-  "datatQualityDimension": "Accuracy"
+  "dataQualityDimension": "Accuracy"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeInSet.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeInSet.json
@ -24,5 +24,5 @@
  ],
  "supportsRowLevelPassedFailed": true,
  "provider": "system",
-  "datatQualityDimension": "Validity"
+  "dataQualityDimension": "Validity"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotInSet.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotInSet.json
@ -17,5 +17,5 @@
  ],
  "supportsRowLevelPassedFailed": true,
  "provider": "system",
-  "datatQualityDimension": "Validity"
+  "dataQualityDimension": "Validity"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotNull.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotNull.json
@ -8,5 +8,5 @@
  "supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"],
  "supportsRowLevelPassedFailed": true,
  "provider": "system",
-  "datatQualityDimension": "Completeness"
+  "dataQualityDimension": "Completeness"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeUnique.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeUnique.json
@ -8,5 +8,5 @@
  "supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"],
  "supportsRowLevelPassedFailed": true,
  "provider": "system",
-  "datatQualityDimension": "Uniqueness"
+  "dataQualityDimension": "Uniqueness"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToMatchRegex.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToMatchRegex.json
@ -17,5 +17,5 @@
  ],
  "supportsRowLevelPassedFailed": true,
  "provider": "system",
-  "datatQualityDimension": "Validity"
+  "dataQualityDimension": "Validity"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToNotMatchRegex.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToNotMatchRegex.json
@ -17,5 +17,5 @@
  ],
  "supportsRowLevelPassedFailed": true,
  "provider": "system",
-  "datatQualityDimension": "Validity"
+  "dataQualityDimension": "Validity"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToBeBetween.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToBeBetween.json
@ -28,5 +28,5 @@
    }
  ],
  "provider": "system",
-  "datatQualityDimension": "Integrity"
+  "dataQualityDimension": "Integrity"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToEqual.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToEqual.json
@ -15,5 +15,5 @@
    }
  ],
  "provider": "system",
-  "datatQualityDimension": "Integrity"
+  "dataQualityDimension": "Integrity"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/tableColumnNameToExist.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/tableColumnNameToExist.json
@ -15,6 +15,6 @@
      }
    ],
    "provider": "system",
-    "datatQualityDimension": "Integrity"
+    "dataQualityDimension": "Integrity"
  }
  
--- a/openmetadata-service/src/main/resources/json/data/tests/tableColumnToMatchSet.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/tableColumnToMatchSet.json
@ -21,6 +21,6 @@
      }
    ],
    "provider": "system",
-    "datatQualityDimension": "Integrity"
+    "dataQualityDimension": "Integrity"
  }
  
--- a/openmetadata-service/src/main/resources/json/data/tests/tableCustomSQLQuery.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/tableCustomSQLQuery.json
@ -30,6 +30,6 @@
      }
    ],
    "provider": "system",
-    "datatQualityDimension": "SQL"
+    "dataQualityDimension": "SQL"
  }
  
--- a/openmetadata-service/src/main/resources/json/data/tests/tableDiff.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/tableDiff.json
@ -45,6 +45,6 @@
    }
  ],
  "provider": "system",
-  "datatQualityDimension": "Consistency"
+  "dataQualityDimension": "Consistency"
 }
  
--- a/openmetadata-service/src/main/resources/json/data/tests/tableRowCountToBeBetween.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/tableRowCountToBeBetween.json
@ -29,5 +29,5 @@
  ],
  "supportsDynamicAssertion": true,
  "provider": "system",
-  "datatQualityDimension": "Integrity"
+  "dataQualityDimension": "Integrity"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/tableRowCountToEqual.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/tableRowCountToEqual.json
@ -15,5 +15,5 @@
    }
  ],
  "provider": "system",
-  "datatQualityDimension": "Integrity"
+  "dataQualityDimension": "Integrity"
 }
--- a/openmetadata-service/src/main/resources/json/data/tests/tableRowInsertedCountToBeBetween.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/tableRowInsertedCountToBeBetween.json
@ -51,6 +51,6 @@
      }
    ],
    "provider": "system",
-    "datatQualityDimension": "Integrity"
+    "dataQualityDimension": "Integrity"
  }
  
--- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java
@ -924,6 +924,12 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
    assertNotNull(testCase.getEntityLink());
    assertNotNull(testCase.getName());
    assertNotNull(testCase.getId());
+
+    // Test return only the specified dimension
+    queryParams.clear();
+    queryParams.put("dataQualityDimension", "Completeness");
+    allEntities = listEntitiesFromSearch(queryParams, testCasesNum, 0, ADMIN_AUTH_HEADERS);
+    assertNotEquals(0, allEntities.getData().size());
  }

  @Test
--- a/openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json
+++ b/openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json
@ -162,7 +162,7 @@
        "$ref": "#/definitions/testCaseParameterDefinition"
      }
    },
-    "datatQualityDimension": {
+    "dataQualityDimension": {
      "$ref": "#/definitions/dataQualityDimensions"
    },
    "owner": {