[GEN-876] Implement DQ dimension filter for test cases (#17032)

* fix: typo in key for data dimension

* fix: table test key typo in data dimension

* fix: typo in migration util

* fix: add filter test cases by dimensions in search API

* style: ran java linting

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java

@@ -1068,8 +1068,13 @@ public abstract class EntityRepository<T extends EntityInterface> {
         entityList.add(withHref(uriInfo, entity));
       }
       return new ResultList<>(entityList, offset, limit, total.intValue());
+    } else {
+      SearchClient.SearchResultListMapper results =
+          searchRepository.listWithOffset(
+              searchListFilter, limit, offset, entityType, searchSortFilter, q);
+      total = results.getTotal();
+      return new ResultList<>(entityList, null, limit, total.intValue());
     }
-    throw new IllegalArgumentException("Limit should be greater than 0");
   }
 
   @Transaction

openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v150/MigrationUtil.java

@@ -64,7 +64,7 @@ public class MigrationUtil {
                         testCaseDefinition.getFullyQualifiedName());
                     return;
                   }
-                  testCaseDefinition.setDatatQualityDimension(dimension);
+                  testCaseDefinition.setDataQualityDimension(dimension);
                   collectionDAO.testDefinitionDAO().update(testCaseDefinition);
                 } catch (Exception e) {
                   LOG.warn("Error migrating test case dimension", e);

openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java

@@ -91,7 +91,7 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
 
   static final String FIELDS = "owner,testSuite,testDefinition,testSuites,incidentId,domain,tags";
   static final String SEARCH_FIELDS_EXCLUDE =
-      "testPlatforms,table,database,databaseSchema,service,testSuite";
+      "testPlatforms,table,database,databaseSchema,service,testSuite,dataQualityDimension";
 
   @Override
   public TestCase addHref(UriInfo uriInfo, TestCase test) {
@@ -316,6 +316,12 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
               schema = @Schema(type = "string"))
           @QueryParam("testPlatforms")
           String testPlatforms,
+      @Parameter(
+              description =
+                  "Filter for test case by data quality dimension (e.g. OpenMetadata, dbt, etc.)",
+              schema = @Schema(type = "string"))
+          @QueryParam("dataQualityDimension")
+          String dataQualityDimension,
       @Parameter(
               description =
                   "Parameter used to filter (inclusive) the test cases by the last execution timestamp (in milliseconds). Must be used in conjunction with `endTimestamp`",
@@ -392,6 +398,7 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
     searchListFilter.addQueryParam("testCaseStatus", status);
     searchListFilter.addQueryParam("testCaseType", type);
     searchListFilter.addQueryParam("testPlatforms", testPlatforms);
+    searchListFilter.addQueryParam("dataQualityDimension", dataQualityDimension);
     searchListFilter.addQueryParam("q", q);
     searchListFilter.addQueryParam("excludeFields", SEARCH_FIELDS_EXCLUDE);
     searchListFilter.addQueryParam("includeFields", includeFields);

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java

@@ -141,6 +141,7 @@ public class SearchListFilter extends Filter<SearchListFilter> {
     String tags = getQueryParam("tags");
     String tier = getQueryParam("tier");
     String serviceName = getQueryParam("serviceName");
+    String dataQualityDimension = getQueryParam("dataQualityDimension");
 
     if (tags != null) {
       String tagsList =
@@ -209,6 +210,11 @@ public class SearchListFilter extends Filter<SearchListFilter> {
           getTimestampFilter("testCaseResult.timestamp", "lte", Long.parseLong(endTimestamp)));
     }
 
+    if (dataQualityDimension != null) {
+      conditions.add(
+          String.format("{\"term\": {\"dataQualityDimension\": \"%s\"}}", dataQualityDimension));
+    }
+
     return addCondition(conditions);
   }
 

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java

@@ -4,11 +4,9 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.UUID;
 import lombok.SneakyThrows;
 import org.openmetadata.schema.tests.TestCase;
 import org.openmetadata.schema.tests.TestDefinition;
-import org.openmetadata.schema.tests.TestPlatform;
 import org.openmetadata.schema.tests.TestSuite;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
@@ -33,6 +31,9 @@ public record TestCaseIndex(TestCase testCase) implements SearchIndex {
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     // Build Index Doc
     List<SearchSuggest> suggest = new ArrayList<>();
+    TestDefinition testDefinition =
+        Entity.getEntity(
+            Entity.TEST_DEFINITION, testCase.getTestDefinition().getId(), "", Include.ALL);
     suggest.add(SearchSuggest.builder().input(testCase.getFullyQualifiedName()).weight(5).build());
     suggest.add(SearchSuggest.builder().input(testCase.getName()).weight(10).build());
     doc.put(
@@ -44,18 +45,13 @@ public record TestCaseIndex(TestCase testCase) implements SearchIndex {
     doc.put("entityType", Entity.TEST_CASE);
     doc.put("owner", getEntityWithDisplayName(testCase.getOwner()));
     doc.put("tags", testCase.getTags());
-    doc.put("testPlatforms", getTestDefinitionPlatforms(testCase.getTestDefinition().getId()));
+    doc.put("testPlatforms", testDefinition.getTestPlatforms());
+    doc.put("dataQualityDimension", testDefinition.getDataQualityDimension());
     doc.put("followers", SearchIndexUtils.parseFollowers(testCase.getFollowers()));
     setParentRelationships(doc, testCase);
     return doc;
   }
 
-  private List<TestPlatform> getTestDefinitionPlatforms(UUID testDefinitionId) {
-    TestDefinition testDefinition =
-        Entity.getEntity(Entity.TEST_DEFINITION, testDefinitionId, "", Include.ALL);
-    return testDefinition.getTestPlatforms();
-  }
-
   private void setParentRelationships(Map<String, Object> doc, TestCase testCase) {
     // denormalize the parent relationships for search
     EntityReference testSuiteEntityReference = testCase.getTestSuite();

openmetadata-service/src/main/resources/elasticsearch/en/test_case_index_mapping.json

@@ -169,6 +169,9 @@
       "entityType": {
         "type": "keyword"
       },
+      "dataQualityDimension": {
+        "type": "keyword"
+      },
       "suggest": {
         "type": "completion",
         "contexts": [

openmetadata-service/src/main/resources/elasticsearch/jp/test_case_index_mapping.json

@@ -192,6 +192,9 @@
       "entityType": {
         "type": "keyword"
       },
+      "dataQualityDimension": {
+        "type": "keyword"
+      },
       "suggest": {
         "type": "completion",
         "contexts": [

openmetadata-service/src/main/resources/elasticsearch/zh/test_case_index_mapping.json

@@ -215,6 +215,9 @@
       "entityType": {
         "type": "keyword"
       },
+      "dataQualityDimension": {
+        "type": "keyword"
+      },
       "suggest": {
         "type": "completion",
         "contexts": [

openmetadata-service/src/main/resources/json/data/tests/columnValueMaxToBeBetween.json

@@ -30,6 +30,6 @@
   ],
   "supportsDynamicAssertion": true,
   "provider": "system",
-  "datatQualityDimension": "Accuracy"
+  "dataQualityDimension": "Accuracy"
 }
   

openmetadata-service/src/main/resources/json/data/tests/columnValueMeanToBeBetween.json

@@ -30,6 +30,6 @@
   ],
   "supportsDynamicAssertion": true,
   "provider": "system",
-  "datatQualityDimension": "Accuracy"
+  "dataQualityDimension": "Accuracy"
 }
   

openmetadata-service/src/main/resources/json/data/tests/columnValueMedianToBeBetween.json

@@ -30,6 +30,6 @@
   ],
   "supportsDynamicAssertion": true,
   "provider": "system",
-  "datatQualityDimension": "Accuracy"
+  "dataQualityDimension": "Accuracy"
 }
   

openmetadata-service/src/main/resources/json/data/tests/columnValueMinToBeBetween.json

@@ -30,6 +30,6 @@
     ],
     "supportsDynamicAssertion": true,
     "provider": "system",
-    "datatQualityDimension": "Accuracy"
+    "dataQualityDimension": "Accuracy"
   }
   

openmetadata-service/src/main/resources/json/data/tests/columnValueStdDevToBeBetween.json

@@ -30,6 +30,6 @@
     ],
     "supportsDynamicAssertion": true,
     "provider": "system",
-    "datatQualityDimension": "Accuracy"
+    "dataQualityDimension": "Accuracy"
   }
   

openmetadata-service/src/main/resources/json/data/tests/columnValuesLengthsToBeBetween.json

@@ -31,5 +31,5 @@
   "supportsRowLevelPassedFailed": true,
   "supportsDynamicAssertion": true,
   "provider": "system",
-  "datatQualityDimension": "Accuracy"
+  "dataQualityDimension": "Accuracy"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesMissingCountToBeEqual.json

@@ -22,5 +22,5 @@
     }
   ],
   "provider": "system",
-  "datatQualityDimension": "Completeness"
+  "dataQualityDimension": "Completeness"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesSumToBeBetween.json

@@ -30,6 +30,6 @@
     ],
     "supportsDynamicAssertion": true,
     "provider": "system",
-    "datatQualityDimension": "Accuracy"
+    "dataQualityDimension": "Accuracy"
   }
   

openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeBetween.json

@@ -31,5 +31,5 @@
   "supportsRowLevelPassedFailed": true,
   "supportsDynamicAssertion": true,
   "provider": "system",
-  "datatQualityDimension": "Accuracy"
+  "dataQualityDimension": "Accuracy"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeInSet.json

@@ -24,5 +24,5 @@
   ],
   "supportsRowLevelPassedFailed": true,
   "provider": "system",
-  "datatQualityDimension": "Validity"
+  "dataQualityDimension": "Validity"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotInSet.json

@@ -17,5 +17,5 @@
   ],
   "supportsRowLevelPassedFailed": true,
   "provider": "system",
-  "datatQualityDimension": "Validity"
+  "dataQualityDimension": "Validity"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotNull.json

@@ -8,5 +8,5 @@
   "supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"],
   "supportsRowLevelPassedFailed": true,
   "provider": "system",
-  "datatQualityDimension": "Completeness"
+  "dataQualityDimension": "Completeness"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeUnique.json

@@ -8,5 +8,5 @@
   "supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"],
   "supportsRowLevelPassedFailed": true,
   "provider": "system",
-  "datatQualityDimension": "Uniqueness"
+  "dataQualityDimension": "Uniqueness"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesToMatchRegex.json

@@ -17,5 +17,5 @@
   ],
   "supportsRowLevelPassedFailed": true,
   "provider": "system",
-  "datatQualityDimension": "Validity"
+  "dataQualityDimension": "Validity"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesToNotMatchRegex.json

@@ -17,5 +17,5 @@
   ],
   "supportsRowLevelPassedFailed": true,
   "provider": "system",
-  "datatQualityDimension": "Validity"
+  "dataQualityDimension": "Validity"
 }

openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToBeBetween.json

@@ -28,5 +28,5 @@
     }
   ],
   "provider": "system",
-  "datatQualityDimension": "Integrity"
+  "dataQualityDimension": "Integrity"
 }

openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToEqual.json

@@ -15,5 +15,5 @@
     }
   ],
   "provider": "system",
-  "datatQualityDimension": "Integrity"
+  "dataQualityDimension": "Integrity"
 }

openmetadata-service/src/main/resources/json/data/tests/tableColumnNameToExist.json

@@ -15,6 +15,6 @@
       }
     ],
     "provider": "system",
-    "datatQualityDimension": "Integrity"
+    "dataQualityDimension": "Integrity"
   }
   

openmetadata-service/src/main/resources/json/data/tests/tableColumnToMatchSet.json

@@ -21,6 +21,6 @@
       }
     ],
     "provider": "system",
-    "datatQualityDimension": "Integrity"
+    "dataQualityDimension": "Integrity"
   }
   

openmetadata-service/src/main/resources/json/data/tests/tableCustomSQLQuery.json

@@ -30,6 +30,6 @@
       }
     ],
     "provider": "system",
-    "datatQualityDimension": "SQL"
+    "dataQualityDimension": "SQL"
   }
   

openmetadata-service/src/main/resources/json/data/tests/tableDiff.json

@@ -45,6 +45,6 @@
     }
   ],
   "provider": "system",
-  "datatQualityDimension": "Consistency"
+  "dataQualityDimension": "Consistency"
 }
   

openmetadata-service/src/main/resources/json/data/tests/tableRowCountToBeBetween.json

@@ -29,5 +29,5 @@
   ],
   "supportsDynamicAssertion": true,
   "provider": "system",
-  "datatQualityDimension": "Integrity"
+  "dataQualityDimension": "Integrity"
 }

openmetadata-service/src/main/resources/json/data/tests/tableRowCountToEqual.json

@@ -15,5 +15,5 @@
     }
   ],
   "provider": "system",
-  "datatQualityDimension": "Integrity"
+  "dataQualityDimension": "Integrity"
 }

openmetadata-service/src/main/resources/json/data/tests/tableRowInsertedCountToBeBetween.json

@@ -51,6 +51,6 @@
       }
     ],
     "provider": "system",
-    "datatQualityDimension": "Integrity"
+    "dataQualityDimension": "Integrity"
   }
   

openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java

@@ -924,6 +924,12 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
     assertNotNull(testCase.getEntityLink());
     assertNotNull(testCase.getName());
     assertNotNull(testCase.getId());
+
+    // Test return only the specified dimension
+    queryParams.clear();
+    queryParams.put("dataQualityDimension", "Completeness");
+    allEntities = listEntitiesFromSearch(queryParams, testCasesNum, 0, ADMIN_AUTH_HEADERS);
+    assertNotEquals(0, allEntities.getData().size());
   }
 
   @Test

openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json

@@ -162,7 +162,7 @@
         "$ref": "#/definitions/testCaseParameterDefinition"
       }
     },
-    "datatQualityDimension": {
+    "dataQualityDimension": {
       "$ref": "#/definitions/dataQualityDimensions"
     },
     "owner": {