diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java index 8f3bd137416..edede39ad54 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java @@ -4,13 +4,17 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import javax.json.JsonArray; +import javax.json.JsonNumber; import javax.json.JsonObject; +import javax.json.JsonString; +import javax.json.JsonValue; import org.openmetadata.schema.tests.DataQualityReport; import org.openmetadata.schema.tests.Datum; import org.openmetadata.schema.tests.type.DataQualityReportMetadata; @@ -191,10 +195,17 @@ public final class SearchIndexUtils { .forEach( bucket -> { JsonObject bucketObject = (JsonObject) bucket; - Optional bucketKey = Optional.of(bucketObject.getString("key")); + Optional val = Optional.of(bucketObject.get("key")); - bucketKey.ifPresentOrElse( - s -> nodeData.put(dimensions.get(0), s), + val.ifPresentOrElse( + s -> { + switch (s.getValueType()) { + case NUMBER -> nodeData.put( + dimensions.get(0), String.valueOf((JsonNumber) s)); + default -> nodeData.put( + dimensions.get(0), ((JsonString) s).getString()); + } + }, () -> nodeData.put(dimensions.get(0), null)); // Traverse the next level of the aggregation tree. @@ -257,21 +268,36 @@ public final class SearchIndexUtils { Map aggregationMap = new HashMap<>(); String[] parts = nested[i].split(":"); - for (String part : parts) { - String[] kvPairs = part.split("="); - if (kvPairs[0].equals("field")) { - aggregationString - .append("\"") - .append(kvPairs[0]) - .append("\":\"") - .append(kvPairs[1]) - .append("\""); + Iterator partsIterator = Arrays.stream(parts).iterator(); + + while (partsIterator.hasNext()) { + String part = partsIterator.next(); + if (!partsIterator.hasNext()) { + // last element = key=value pairs of the aggregation + String[] subParts = part.split("&"); + Arrays.stream(subParts) + .forEach( + subPart -> { + String[] kvPairs = subPart.split("="); + aggregationString + .append("\"") + .append(kvPairs[0]) + .append("\":\"") + .append(kvPairs[1]) + .append("\""); + aggregationMap.put(kvPairs[0], kvPairs[1]); + // add comma if not the last element + if (Arrays.asList(subParts).indexOf(subPart) < subParts.length - 1) + aggregationString.append(","); + }); aggregationString.append("}"); } else { + String[] kvPairs = part.split("="); aggregationString.append("\"").append(kvPairs[1]).append("\":{"); + aggregationMap.put(kvPairs[0], kvPairs[1]); } - aggregationMap.put(kvPairs[0], kvPairs[1]); } + if (i < nested.length - 1) { aggregationString.append(",\"aggs\":{"); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index cad5dd097ec..47989d82a55 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -1009,6 +1009,15 @@ public class ElasticSearchClient implements SearchClient { AggregationBuilders.avg(key).field(avgAggregation.getString("field")); aggregationBuilders.add(avgAggregationBuilder); break; + case "date_histogram": + JsonObject dateHistogramAggregation = aggregation.getJsonObject(aggregationType); + String calendarInterval = dateHistogramAggregation.getString("calendar_interval"); + DateHistogramAggregationBuilder dateHistogramAggregationBuilder = + AggregationBuilders.dateHistogram(key) + .field(dateHistogramAggregation.getString("field")) + .calendarInterval(new DateHistogramInterval(calendarInterval)); + aggregationBuilders.add(dateHistogramAggregationBuilder); + break; case "nested": JsonObject nestedAggregation = aggregation.getJsonObject("nested"); AggregationBuilder nestedAggregationBuilder = diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index 1ffe67e5df0..56439482911 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -28,6 +28,7 @@ import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.schema.type.ColumnDataType.BIGINT; import static org.openmetadata.schema.type.MetadataOperation.EDIT_TESTS; import static org.openmetadata.service.Entity.ADMIN_USER_NAME; +import static org.openmetadata.service.Entity.getSearchRepository; import static org.openmetadata.service.exception.CatalogExceptionMessage.permissionNotAllowed; import static org.openmetadata.service.jdbi3.TestCaseRepository.FAILED_ROWS_SAMPLE_EXTENSION; import static org.openmetadata.service.security.SecurityUtil.authHeaders; @@ -75,6 +76,7 @@ import org.openmetadata.schema.api.tests.CreateTestCaseResolutionStatus; import org.openmetadata.schema.api.tests.CreateTestSuite; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.feed.Thread; +import org.openmetadata.schema.tests.DataQualityReport; import org.openmetadata.schema.tests.ResultSummary; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestCaseParameterValue; @@ -101,6 +103,8 @@ import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; import org.openmetadata.service.resources.feeds.FeedResourceTest; import org.openmetadata.service.resources.feeds.MessageParser; +import org.openmetadata.service.search.SearchIndexUtils; +import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.indexes.TestCaseIndex; import org.openmetadata.service.search.models.IndexMapping; import org.openmetadata.service.util.JsonUtils; @@ -2751,6 +2755,37 @@ public class TestCaseResourceTest extends EntityResourceTest aggregationString = + SearchIndexUtils.buildAggregationString(aggregationQuery); + DataQualityReport dataQualityReport = + searchRepository.genericAggregation(null, "testCaseResult", aggregationString); + assertNotNull(dataQualityReport.getData()); + } + @Test void createTestCaseResults_wrongTs(TestInfo testInfo) throws IOException, HttpResponseException { CreateTestCase create = createRequest(testInfo); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java index 8756a499a9b..a29aa7d93b6 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java @@ -447,6 +447,22 @@ public class SearchIndexResourceTest extends EntityResourceTest m = datum.getAdditionalProperties(); assertTrue(m.containsKey("fullyQualifiedName")); }); + + aggregationQuery = + "bucketName=dates:aggType=date_histogram:field=timestamp&calendar_interval=1d,bucketName=dimesion:aggType=terms:field=testDefinition.dataQualityDimension"; + aggregationString = SearchIndexUtils.buildAggregationString(aggregationQuery); + dataQualityReport = + searchRepository.genericAggregation(null, "testCaseResult", aggregationString); } @Override diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ServiceBaseClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ServiceBaseClass.ts index f1e9a4a4206..c5a408a4e7c 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ServiceBaseClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ServiceBaseClass.ts @@ -176,10 +176,10 @@ class ServiceBaseClass { .getByLabel('Ingestions') .getByTestId('loader') .waitFor({ state: 'detached' }); - + // need manual wait to settle down the deployed pipeline, before triggering the pipeline await page.waitForTimeout(2000); - + await page.getByTestId('more-actions').first().click(); await page.getByTestId('run-button').click();