diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ElasticSearchBulkSink.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ElasticSearchBulkSink.java index bc38f889002..37fde10fa9b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ElasticSearchBulkSink.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ElasticSearchBulkSink.java @@ -206,7 +206,7 @@ public class ElasticSearchBulkSink implements BulkSink { if (!entities.isEmpty() && entities.get(0) instanceof EntityTimeSeriesInterface) { List tsEntities = (List) entities; for (EntityTimeSeriesInterface entity : tsEntities) { - addTimeSeriesEntity(entity, indexName); + addTimeSeriesEntity(entity, indexName, entityType); } } else { List entityInterfaces = (List) entities; @@ -256,8 +256,10 @@ public class ElasticSearchBulkSink implements BulkSink { } } - private void addTimeSeriesEntity(EntityTimeSeriesInterface entity, String indexName) { - String json = JsonUtils.pojoToJson(entity); + private void addTimeSeriesEntity( + EntityTimeSeriesInterface entity, String indexName, String entityType) { + Object searchIndexDoc = Entity.buildSearchIndex(entityType, entity).buildSearchIndexDoc(); + String json = JsonUtils.pojoToJson(searchIndexDoc); String docId = entity.getId().toString(); IndexRequest indexRequest = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/OpenSearchBulkSink.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/OpenSearchBulkSink.java index 4dc3e7ad381..0101ac1f5ce 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/OpenSearchBulkSink.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/OpenSearchBulkSink.java @@ -206,7 +206,7 @@ public class OpenSearchBulkSink implements BulkSink { if (!entities.isEmpty() && entities.getFirst() instanceof EntityTimeSeriesInterface) { List tsEntities = (List) entities; for (EntityTimeSeriesInterface entity : tsEntities) { - addTimeSeriesEntity(entity, indexName); + addTimeSeriesEntity(entity, indexName, entityType); } } else { List entityInterfaces = (List) entities; @@ -256,8 +256,10 @@ public class OpenSearchBulkSink implements BulkSink { } } - private void addTimeSeriesEntity(EntityTimeSeriesInterface entity, String indexName) { - String json = JsonUtils.pojoToJson(entity); + private void addTimeSeriesEntity( + EntityTimeSeriesInterface entity, String indexName, String entityType) { + Object searchIndexDoc = Entity.buildSearchIndex(entityType, entity).buildSearchIndexDoc(); + String json = JsonUtils.pojoToJson(searchIndexDoc); String docId = entity.getId().toString(); IndexRequest indexRequest = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java index 9247ad5e04c..3dc9b3179f9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java @@ -117,13 +117,15 @@ public class TestSuiteRepository extends EntityRepository { UPDATE_FIELDS); quoteFqn = false; supportsSearch = true; + fieldFetchers.put("summary", this::fetchAndSetTestCaseResultSummary); + fieldFetchers.put("pipelines", this::fetchAndSetIngestionPipelines); } @Override public void setFields(TestSuite entity, EntityUtil.Fields fields) { entity.setPipelines( fields.contains("pipelines") ? getIngestionPipelines(entity) : entity.getPipelines()); - entity.setTests(fields.contains(UPDATE_FIELDS) ? getTestCases(entity) : entity.getTests()); + entity.setTests(fields.contains("tests") ? getTestCases(entity) : entity.getTests()); entity.setTestCaseResultSummary( fields.contains("summary") ? getResultSummary(entity.getId()) @@ -169,7 +171,9 @@ public class TestSuiteRepository extends EntityRepository { } var testSuiteIds = testSuites.stream().map(ts -> ts.getId().toString()).toList(); var records = - daoCollection.relationshipDAO().findFromBatch(testSuiteIds, Relationship.HAS.ordinal()); + daoCollection + .relationshipDAO() + .findToBatch(testSuiteIds, Relationship.CONTAINS.ordinal(), TEST_SUITE, TEST_CASE); if (records.isEmpty()) { return Map.of(); } @@ -411,6 +415,40 @@ public class TestSuiteRepository extends EntityRepository { } } + private void fetchAndSetTestCaseResultSummary( + List testSuites, EntityUtil.Fields fields) { + if (!fields.contains("summary") || testSuites == null || testSuites.isEmpty()) { + return; + } + + Map> testCaseResultSummaryMap = + testSuites.stream() + .collect( + Collectors.toMap( + TestSuite::getId, testSuite -> getResultSummary(testSuite.getId()))); + + Map testSummaryMap = + testCaseResultSummaryMap.entrySet().stream() + .collect( + Collectors.toMap(Map.Entry::getKey, entry -> getTestSummary(entry.getValue()))); + + setFieldFromMap( + true, testSuites, testCaseResultSummaryMap, TestSuite::setTestCaseResultSummary); + + setFieldFromMap(true, testSuites, testSummaryMap, TestSuite::setSummary); + } + + protected void fetchAndSetIngestionPipelines(List entities, EntityUtil.Fields fields) { + if (!fields.contains("pipelines") || entities == null || entities.isEmpty()) { + return; + } + + Map> ingestionPipelineMap = + entities.stream() + .collect(Collectors.toMap(EntityInterface::getId, this::getIngestionPipelines)); + setFieldFromMap(true, entities, ingestionPipelineMap, TestSuite::setPipelines); + } + @SneakyThrows private List getResultSummary(UUID testSuiteId) { List resultSummaries = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticTermsAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticTermsAggregations.java index c1f3bf45038..de7b6180b9a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticTermsAggregations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticTermsAggregations.java @@ -35,6 +35,9 @@ public class ElasticTermsAggregations implements ElasticAggregations { IncludeExclude includeExclude = new IncludeExclude(includes, null); termsAggregationBuilder.includeExclude(includeExclude); } + if (params.get("missing") != null) { + termsAggregationBuilder.missing(params.get("missing")); + } setElasticAggregationBuilder(termsAggregationBuilder); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenTermsAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenTermsAggregations.java index 269c3b9212e..7477b884d92 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenTermsAggregations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenTermsAggregations.java @@ -35,6 +35,9 @@ public class OpenTermsAggregations implements OpenAggregations { IncludeExclude includeExclude = new IncludeExclude(includes, null); termsAggregationBuilder.includeExclude(includeExclude); } + if (params.get("missing") != null) { + termsAggregationBuilder.missing(params.get("missing")); + } setElasticAggregationBuilder(termsAggregationBuilder); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java index 6cc9921d9e1..4cf4260d520 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java @@ -3,6 +3,7 @@ package org.openmetadata.service.resources.dqtests; import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -50,6 +51,7 @@ import org.openmetadata.schema.metadataIngestion.SourceConfig; import org.openmetadata.schema.metadataIngestion.TestSuitePipeline; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestSuite; +import org.openmetadata.schema.tests.type.TestCaseResult; import org.openmetadata.schema.tests.type.TestCaseStatus; import org.openmetadata.schema.tests.type.TestSummary; import org.openmetadata.schema.type.Column; @@ -65,6 +67,7 @@ import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.resources.services.ingestionpipelines.IngestionPipelineResourceTest; import org.openmetadata.service.resources.teams.TeamResourceTest; import org.openmetadata.service.resources.teams.UserResourceTest; +import org.openmetadata.service.security.SecurityUtil; import org.openmetadata.service.util.ResultList; import org.openmetadata.service.util.TestUtils; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; @@ -1030,4 +1033,135 @@ public class TestSuiteResourceTest extends EntityResourceTest testCaseResultsQueryParams = new HashMap<>(); + testCaseResultsQueryParams.put("testCaseId", testCase.getId().toString()); + testCaseResultsQueryParams.put("fields", "testCase,testDefinition,testCaseStatus"); + ResultList testCaseResultsBeforeReindex = + testCaseResourceTest.listTestCaseResultsFromSearch( + testCaseResultsQueryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + assertNotNull(testCaseResultsBeforeReindex); + assertFalse(testCaseResultsBeforeReindex.getData().isEmpty()); + TestCaseResult testCaseResultBeforeReindex = testCaseResultsBeforeReindex.getData().getFirst(); + + // 4. Fetch the test suite linked to the table using the search endpoint (before reindex) + Map queryParams = new HashMap<>(); + queryParams.put("fullyQualifiedName", testSuite.getFullyQualifiedName()); + queryParams.put("fields", "tests,testCaseResultSummary"); + ResultList testSuitesBeforeReindex = + listEntitiesFromSearch(queryParams, 10, 0, ADMIN_AUTH_HEADERS); + assertNotNull(testSuitesBeforeReindex); + assertFalse(testSuitesBeforeReindex.getData().isEmpty()); + TestSuite testSuiteBeforeReindex = testSuitesBeforeReindex.getData().getFirst(); + + // 5. Trigger an Elasticsearch index refresh for the test suite entity (simulating a reindex) + postTriggerSearchIndexingApp(ADMIN_AUTH_HEADERS); + + // Wait for reindexing to complete + Thread.sleep(5000); + + // 6. Fetch the test suite again using the search endpoint (after reindex) + ResultList testSuitesAfterReindex = + listEntitiesFromSearch(queryParams, 10, 0, ADMIN_AUTH_HEADERS); + assertNotNull(testSuitesAfterReindex); + assertTrue(testSuitesAfterReindex.getData().size() > 0); + TestSuite testSuiteAfterReindex = testSuitesAfterReindex.getData().get(0); + + // 6a. Verify test case results are still available via search endpoint after reindex + ResultList testCaseResultsAfterReindex = + testCaseResourceTest.listTestCaseResultsFromSearch( + testCaseResultsQueryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + assertNotNull(testCaseResultsAfterReindex); + assertFalse(testCaseResultsAfterReindex.getData().isEmpty()); + TestCaseResult testCaseResultAfterReindex = testCaseResultsAfterReindex.getData().getFirst(); + + // Compare test case results before and after reindex + assertEquals( + testCaseResultBeforeReindex.getTestCaseStatus(), + testCaseResultAfterReindex.getTestCaseStatus()); + assertEquals(testCaseResultBeforeReindex.getResult(), testCaseResultAfterReindex.getResult()); + assertEquals( + testCaseResultBeforeReindex.getTimestamp(), testCaseResultAfterReindex.getTimestamp()); + assertNotNull(testCaseResultAfterReindex.getTestCase()); + assertEquals(testCase.getId(), testCaseResultAfterReindex.getTestCase().getId()); + + // Compare testDefinition + assertEquals( + testCaseResultBeforeReindex.getTestDefinition(), + testCaseResultAfterReindex.getTestDefinition()); + + // 7. Compare the test suite results from before and after the reindex to ensure they are + // identical + assertEquals(testSuiteBeforeReindex.getId(), testSuiteAfterReindex.getId()); + assertEquals(testSuiteBeforeReindex.getName(), testSuiteAfterReindex.getName()); + assertEquals( + testSuiteBeforeReindex.getFullyQualifiedName(), + testSuiteAfterReindex.getFullyQualifiedName()); + assertEquals(testSuiteBeforeReindex.getDescription(), testSuiteAfterReindex.getDescription()); + + // Assert that every test ID is in both before and after testSuite + for (EntityReference testRef : testSuiteBeforeReindex.getTests()) { + assertTrue( + testSuiteAfterReindex.getTests().stream() + .allMatch(t -> t.getId().equals(testRef.getId())), + "Test ID " + testRef.getId() + " should be present in testSuite after reindex"); + } + + // Compare test cases + assertEquals(testSuiteBeforeReindex.getTests().size(), testSuiteAfterReindex.getTests().size()); + if (testSuiteBeforeReindex.getTests() != null && testSuiteAfterReindex.getTests() != null) { + verifyTestCases(testSuiteBeforeReindex.getTests(), testSuiteAfterReindex.getTests()); + } + + // Compare test case result summaries + assertEquals( + testSuiteBeforeReindex.getTestCaseResultSummary(), + testSuiteAfterReindex.getTestCaseResultSummary()); + } + + private void postTriggerSearchIndexingApp(Map authHeaders) throws IOException { + WebTarget target = getResource("apps/trigger").path("SearchIndexingApplication"); + Response response = + SecurityUtil.addHeaders(target, authHeaders) + .post(jakarta.ws.rs.client.Entity.json(Map.of())); + TestUtils.readResponse(response, Response.Status.OK.getStatusCode()); + } }