diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py index 4fdeb133349..94853f61cdb 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py @@ -15,8 +15,9 @@ To be used by OpenMetadata class """ from datetime import datetime, timezone -from typing import List, Optional +from typing import List, Optional, Type, Union from urllib.parse import quote +from uuid import UUID from metadata.generated.schema.api.tests.createLogicalTestCases import ( CreateLogicalTestCases, @@ -37,6 +38,7 @@ from metadata.generated.schema.tests.testDefinition import ( from metadata.generated.schema.tests.testSuite import TestSuite from metadata.ingestion.models.encoders import show_secrets_encoder from metadata.ingestion.ometa.client import REST +from metadata.ingestion.ometa.utils import model_str from metadata.utils.logger import ometa_logger logger = ometa_logger() @@ -241,6 +243,25 @@ class OMetaTestsMixin: return entity_class.parse_obj(resp) + def delete_executable_test_suite( + self, + entity: Type[TestSuite], + entity_id: Union[str, UUID], + recursive: bool = False, + hard_delete: bool = False, + ) -> None: + """Delete executable test suite + + Args: + entity_id (str): test suite ID + recursive (bool, optional): delete children if true + hard_delete (bool, optional): hard delete if true + """ + url = f"{self.get_suffix(entity)}/executable/{model_str(entity_id)}" + url += f"?recursive={str(recursive).lower()}" + url += f"&hardDelete={str(hard_delete).lower()}" + self.client.delete(url) + def add_logical_test_cases(self, data: CreateLogicalTestCases) -> None: """Add logical test cases to a test suite diff --git a/ingestion/tests/integration/ometa/test_ometa_test_suite.py b/ingestion/tests/integration/ometa/test_ometa_test_suite.py index 62529c65d44..6b196536db2 100644 --- a/ingestion/tests/integration/ometa/test_ometa_test_suite.py +++ b/ingestion/tests/integration/ometa/test_ometa_test_suite.py @@ -168,7 +168,7 @@ class OMetaTestSuiteTest(TestCase): @classmethod def tearDownClass(cls) -> None: - cls.metadata.delete( + cls.metadata.delete_executable_test_suite( entity=TestSuite, entity_id=cls.test_suite.id, recursive=True, diff --git a/ingestion/tests/integration/test_suite/test_workflow.py b/ingestion/tests/integration/test_suite/test_workflow.py index 1bef851ece6..f01bf93e3f3 100644 --- a/ingestion/tests/integration/test_suite/test_workflow.py +++ b/ingestion/tests/integration/test_suite/test_workflow.py @@ -23,7 +23,6 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata OpenMetadataConnection, ) from metadata.generated.schema.tests.testCase import TestCase -from metadata.generated.schema.tests.testDefinition import TestDefinition from metadata.generated.schema.tests.testSuite import TestSuite from metadata.ingestion.ometa.ometa_api import OpenMetadata @@ -76,7 +75,7 @@ class TestSuiteWorkflowTests(unittest.TestCase): hard_delete=True, ) for test_suite_id in self.test_suite_ids: - self.metadata.delete( + self.metadata.delete_executable_test_suite( entity=TestSuite, entity_id=test_suite_id, recursive=True, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchEventPublisher.java b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchEventPublisher.java index ad3c174f33d..2301f6b8ae8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchEventPublisher.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchEventPublisher.java @@ -15,6 +15,8 @@ package org.openmetadata.service.elasticsearch; +import static org.openmetadata.schema.type.EventType.ENTITY_DELETED; +import static org.openmetadata.schema.type.EventType.ENTITY_UPDATED; import static org.openmetadata.service.Entity.ADMIN_USER_NAME; import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; import static org.openmetadata.service.Entity.FIELD_USAGE_SUMMARY; @@ -51,15 +53,18 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.index.reindex.UpdateByQueryRequest; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.api.CreateEventPublisherJob; import org.openmetadata.schema.entity.classification.Classification; import org.openmetadata.schema.entity.classification.Tag; @@ -80,11 +85,14 @@ import org.openmetadata.schema.system.EventPublisherJob; import org.openmetadata.schema.system.EventPublisherJob.Status; import org.openmetadata.schema.system.Failure; import org.openmetadata.schema.system.FailureDetails; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.EventType; import org.openmetadata.schema.type.FieldChange; +import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.UsageDetails; import org.openmetadata.service.Entity; @@ -178,6 +186,12 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher { case Entity.CLASSIFICATION: updateClassification(event); break; + case Entity.TEST_CASE: + updateTestCase(event); + break; + case Entity.TEST_SUITE: + updateTestSuite(event); + break; default: LOG.warn("Ignoring Entity Type {}", entityType); } @@ -427,6 +441,106 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher { } } + private void updateTestSuite(ChangeEvent event) throws IOException { + ElasticSearchIndexType indexType = ElasticSearchIndexDefinition.getIndexMappingByEntityType(Entity.TEST_CASE); + TestSuite testSuite = (TestSuite) event.getEntity(); + UUID testSuiteId = testSuite.getId(); + + if (event.getEventType() == ENTITY_DELETED) { + if (testSuite.getExecutable()) { + DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(indexType.indexName); + deleteByQueryRequest.setQuery(new MatchQueryBuilder("testSuite.id", testSuiteId.toString())); + deleteEntityFromElasticSearchByQuery(deleteByQueryRequest); + } else { + UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexType.indexName); + updateByQueryRequest.setQuery(new MatchQueryBuilder("testSuite.id", testSuiteId.toString())); + String scriptTxt = + "for (int i = 0; i < ctx._source.testSuite.length; i++) { if (ctx._source.testSuite[i].id == '%s') { ctx._source.testSuite.remove(i) }}"; + Script script = + new Script( + ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, String.format(scriptTxt, testSuiteId), new HashMap<>()); + updateByQueryRequest.setScript(script); + updateElasticSearchByQuery(updateByQueryRequest); + } + } + } + + private void updateTestCase(ChangeEvent event) throws IOException { + ElasticSearchIndexType indexType = ElasticSearchIndexDefinition.getIndexMappingByEntityType(Entity.TEST_CASE); + // creating a new test case will return a TestCase entity while bulk adding test cases will return + // the logical test suite entity with the newly added test cases + EntityInterface entityInterface = (EntityInterface) event.getEntity(); + if (entityInterface instanceof TestCase) { + processTestCase((TestCase) entityInterface, event, indexType); + } else { + addTestCaseFromLogicalTestSuite((TestSuite) entityInterface, event, indexType); + } + } + + private void addTestCaseFromLogicalTestSuite(TestSuite testSuite, ChangeEvent event, ElasticSearchIndexType indexType) + throws IOException { + // Process creation of test cases (linked to a logical test suite) by adding reference to existing test cases + List testCaseReferences = testSuite.getTests(); + TestSuite testSuiteReference = + new TestSuite() + .withId(testSuite.getId()) + .withName(testSuite.getName()) + .withDisplayName(testSuite.getDisplayName()) + .withDescription(testSuite.getDescription()) + .withFullyQualifiedName(testSuite.getFullyQualifiedName()) + .withDeleted(testSuite.getDeleted()) + .withHref(testSuite.getHref()) + .withExecutable(testSuite.getExecutable()); + Map testSuiteDoc = JsonUtils.getMap(testSuiteReference); + if (event.getEventType() == ENTITY_UPDATED) { + for (EntityReference testcaseReference : testCaseReferences) { + UpdateRequest updateRequest = new UpdateRequest(indexType.indexName, testcaseReference.getId().toString()); + String scripText = "ctx._source.testSuite.add(params)"; + Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scripText, testSuiteDoc); + updateRequest.script(script); + updateElasticSearch(updateRequest); + } + } + } + + private void processTestCase(TestCase testCase, ChangeEvent event, ElasticSearchIndexType indexType) + throws IOException { + // Process creation of test cases (linked to an executable test suite + UpdateRequest updateRequest = new UpdateRequest(indexType.indexName, testCase.getId().toString()); + TestCaseIndex testCaseIndex; + + switch (event.getEventType()) { + case ENTITY_CREATED: + testCaseIndex = new TestCaseIndex((TestCase) event.getEntity()); + updateRequest.doc(JsonUtils.pojoToJson(testCaseIndex.buildESDocForCreate()), XContentType.JSON); + updateRequest.docAsUpsert(true); + updateElasticSearch(updateRequest); + break; + case ENTITY_UPDATED: + testCaseIndex = new TestCaseIndex((TestCase) event.getEntity()); + scriptedUpsert(testCaseIndex.buildESDoc(), updateRequest); + updateElasticSearch(updateRequest); + break; + case ENTITY_SOFT_DELETED: + softDeleteEntity(updateRequest); + updateElasticSearch(updateRequest); + break; + case ENTITY_DELETED: + EntityReference testSuiteReference = ((TestCase) event.getEntity()).getTestSuite(); + TestSuite testSuite = Entity.getEntity(Entity.TEST_SUITE, testSuiteReference.getId(), "", Include.ALL); + if (testSuite.getExecutable()) { + // Delete the test case from the index if deleted from an executable test suite + DeleteRequest deleteRequest = new DeleteRequest(indexType.indexName, event.getEntityId().toString()); + deleteEntityFromElasticSearch(deleteRequest); + } else { + // for non-executable test suites, simply remove the testSuite from the testCase and update the index + scriptedDeleteTestCase(updateRequest, testSuite.getId()); + updateElasticSearch(updateRequest); + } + break; + } + } + private void updateTag(ChangeEvent event) throws IOException { UpdateRequest updateRequest = new UpdateRequest(ElasticSearchIndexType.TAG_SEARCH_INDEX.indexName, event.getEntityId().toString()); @@ -633,6 +747,15 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher { updateRequest.scriptedUpsert(true); } + private void scriptedDeleteTestCase(UpdateRequest updateRequest, UUID testSuiteId) { + // Remove logical test suite from test case `testSuite` field + String scriptTxt = + "for (int i = 0; i < ctx._source.testSuite.length; i++) { if (ctx._source.testSuite[i].id == '%s') { ctx._source.testSuite.remove(i) }}"; + scriptTxt = String.format(scriptTxt, testSuiteId); + Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptTxt, new HashMap<>()); + updateRequest.script(script); + } + private void softDeleteEntity(UpdateRequest updateRequest) { String scriptTxt = "ctx._source.deleted=true"; Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptTxt, new HashMap<>()); @@ -646,6 +769,13 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher { } } + private void updateElasticSearchByQuery(UpdateByQueryRequest updateByQueryRequest) throws IOException { + if (updateByQueryRequest != null) { + LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateByQueryRequest); + client.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT); + } + } + private void deleteEntityFromElasticSearch(DeleteRequest deleteRequest) throws IOException { if (deleteRequest != null) { LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, deleteRequest); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexDefinition.java b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexDefinition.java index b7eb6b826e9..ac1809caf65 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexDefinition.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexDefinition.java @@ -99,6 +99,8 @@ public class ElasticSearchIndexDefinition { TAG_SEARCH_INDEX(Entity.TAG, "tag_search_index", "/elasticsearch/%s/tag_index_mapping.json"), ENTITY_REPORT_DATA_INDEX( ENTITY_REPORT_DATA, "entity_report_data_index", "/elasticsearch/entity_report_data_index.json"), + TEST_CASE_SEARCH_INDEX( + Entity.TEST_CASE, "test_case_search_index", "/elasticsearch/%s/test_case_index_mapping.json"), WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA_INDEX( Entity.WEB_ANALYTIC_EVENT, "web_analytic_entity_view_report_data_index", @@ -270,6 +272,8 @@ public class ElasticSearchIndexDefinition { return ElasticSearchIndexType.CONTAINER_SEARCH_INDEX; } else if (type.equalsIgnoreCase(Entity.QUERY)) { return ElasticSearchIndexType.QUERY_SEARCH_INDEX; + } else if (type.equalsIgnoreCase(Entity.TEST_SUITE) || type.equalsIgnoreCase(Entity.TEST_CASE)) { + return ElasticSearchIndexType.TEST_CASE_SEARCH_INDEX; } throw new EventPublisherException("Failed to find index doc for type " + type); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/TestCaseIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/TestCaseIndex.java new file mode 100644 index 00000000000..c1fff3b00b0 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/TestCaseIndex.java @@ -0,0 +1,48 @@ +package org.openmetadata.service.elasticsearch; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.tests.TestSuite; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.util.JsonUtils; + +public class TestCaseIndex implements ElasticSearchIndex { + TestCase testCase; + + public TestCaseIndex(TestCase testCase) { + this.testCase = testCase; + } + + public Map buildESDoc() { + return JsonUtils.getMap(testCase); + } + + public Map buildESDocForCreate() throws IOException { + EntityReference testSuiteEntityReference = testCase.getTestSuite(); + TestSuite testSuite = getTestSuite(testSuiteEntityReference.getId()); + List testSuiteArray = new ArrayList<>(); + testSuiteArray.add(testSuite); + Map doc = JsonUtils.getMap(testCase); + doc.put("testSuite", testSuiteArray); + return doc; + } + + private TestSuite getTestSuite(UUID testSuiteId) throws IOException { + TestSuite testSuite = Entity.getEntity(Entity.TEST_SUITE, testSuiteId, "", Include.ALL); + return new TestSuite() + .withId(testSuite.getId()) + .withName(testSuite.getName()) + .withDisplayName(testSuite.getDisplayName()) + .withDescription(testSuite.getDescription()) + .withFullyQualifiedName(testSuite.getFullyQualifiedName()) + .withDeleted(testSuite.getDeleted()) + .withHref(testSuite.getHref()) + .withExecutable(testSuite.getExecutable()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/util/FormatterUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/util/FormatterUtil.java index d0bc576fed7..1ee63e57da1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/util/FormatterUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/util/FormatterUtil.java @@ -42,6 +42,7 @@ import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.EventType; import org.openmetadata.schema.type.FieldChange; +import org.openmetadata.service.Entity; import org.openmetadata.service.formatter.decorators.MessageDecorator; import org.openmetadata.service.formatter.factory.ParserFactory; import org.openmetadata.service.formatter.field.DefaultFieldFormatter; @@ -268,6 +269,17 @@ public class FormatterUtil { .withEntityFullyQualifiedName(entityFQN); } + // Handles Bulk Add test cases to a logical test suite + if (changeType.equals(RestUtil.LOGICAL_TEST_CASES_ADDED)) { + EntityInterface entityInterface = (EntityInterface) responseContext.getEntity(); + EntityReference entityReference = entityInterface.getEntityReference(); + String entityType = Entity.TEST_CASE; + String entityFQN = entityReference.getFullyQualifiedName(); + return getChangeEvent(updateBy, EventType.ENTITY_UPDATED, entityType, entityInterface) + .withEntity(entityInterface) + .withEntityFullyQualifiedName(entityFQN); + } + // PUT or PATCH operation didn't result in any change if (changeType == null || RestUtil.ENTITY_NO_CHANGE.equals(changeType)) { return null; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 426479321fe..851eeb13b83 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -666,7 +666,14 @@ public interface CollectionDAO { @Bind("relation") int relation, @Bind("json") String json); - @SqlUpdate("INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES ") + @ConnectionAwareSqlUpdate( + value = "INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES ", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES " + + "ON CONFLICT DO NOTHING", + connectionType = POSTGRES) void bulkInsertTo( @BindBeanList( value = "values", @@ -2878,6 +2885,12 @@ public interface CollectionDAO { + "ORDER BY timestamp DESC LIMIT 1") String getLatestExtension(@Bind("entityFQN") String entityFQN, @Bind("extension") String extension); + @SqlQuery( + "SELECT ranked.json FROM (SELECT json, ROW_NUMBER() OVER(PARTITION BY entityFQN ORDER BY timestamp DESC) AS row_num " + + "FROM entity_extension_time_series WHERE entityFQN IN ()) ranked WHERE ranked.row_num = 1") + List getLatestExtensionByFQNs( + @BindList("entityFQNs") List entityFQNs, @Bind("extension") String extension); + @SqlQuery( "SELECT json FROM entity_extension_time_series WHERE extension = :extension " + "ORDER BY timestamp DESC LIMIT 1") diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index c69a89f1abd..948f2cc1b06 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -1449,6 +1449,8 @@ public abstract class EntityRepository { private void updateDescription() throws JsonProcessingException { if (operation.isPut() && !nullOrEmpty(original.getDescription()) && updatedByBot()) { // Revert change to non-empty description if it is being updated by a bot + // This is to prevent bots from overwriting the description. Description need to be + // updated with a PATCH request updated.setDescription(original.getDescription()); return; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java index dc881579376..198ece0d1cc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java @@ -52,6 +52,7 @@ public class ListFilter { condition = addCondition(condition, getWebhookCondition(tableName)); condition = addCondition(condition, getWebhookTypeCondition(tableName)); condition = addCondition(condition, getTestCaseCondition()); + condition = addCondition(condition, getTestSuiteCondition()); return condition.isEmpty() ? "WHERE TRUE" : "WHERE " + condition; } @@ -130,6 +131,29 @@ public class ListFilter { return addCondition(condition1, condition2); } + private String getTestSuiteCondition() { + String testSuiteType = getQueryParam("testSuiteType"); + + if (testSuiteType == null) { + return ""; + } + + switch (testSuiteType) { + case ("executable"): + if (DatasourceConfig.getInstance().isMySQL()) { + return "JSON_UNQUOTE(JSON_EXTRACT(json, '$.executable')) = 'true'"; + } + return "json->>'executable' = 'true'"; + case ("logical"): + if (DatasourceConfig.getInstance().isMySQL()) { + return "JSON_UNQUOTE(JSON_EXTRACT(json, '$.executable')) = 'false'"; + } + return "json->>'executable' = 'false'"; + default: + return ""; + } + } + private String getFqnPrefixCondition(String tableName, String fqnPrefix) { fqnPrefix = escape(fqnPrefix); return tableName == null diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java index 444b9f4f001..847898beaf8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java @@ -5,13 +5,16 @@ import static org.openmetadata.service.Entity.TEST_DEFINITION; import static org.openmetadata.service.Entity.TEST_SUITE; import static org.openmetadata.service.util.RestUtil.ENTITY_NO_CHANGE; import static org.openmetadata.service.util.RestUtil.ENTITY_UPDATED; +import static org.openmetadata.service.util.RestUtil.LOGICAL_TEST_CASES_ADDED; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.UUID; +import java.util.stream.Collectors; import javax.json.JsonPatch; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; @@ -23,6 +26,8 @@ import org.openmetadata.schema.tests.TestCaseParameterValue; import org.openmetadata.schema.tests.TestDefinition; import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.tests.type.TestCaseResult; +import org.openmetadata.schema.tests.type.TestCaseStatus; +import org.openmetadata.schema.tests.type.TestSummary; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.type.EntityReference; @@ -284,20 +289,56 @@ public class TestCaseRepository extends EntityRepository { } } - public RestUtil.PutResponse addTestCasesToLogicalTestSuite(TestSuite testSuite, List testCaseIds) { + public RestUtil.PutResponse addTestCasesToLogicalTestSuite(TestSuite testSuite, List testCaseIds) + throws IOException { bulkAddToRelationship(testSuite.getId(), testCaseIds, TEST_SUITE, TEST_CASE, Relationship.CONTAINS); - return new RestUtil.PutResponse<>( - Response.Status.OK, - testSuite, - String.format(RestUtil.TEST_CASES_ADDED, testCaseIds.size(), testSuite.getName())); + List testCasesEntityReferences = new ArrayList<>(); + for (UUID testCaseId : testCaseIds) { + TestCase testCase = Entity.getEntity(Entity.TEST_CASE, testCaseId, "", Include.ALL); + testCasesEntityReferences.add( + new EntityReference() + .withId(testCase.getId()) + .withName(testCase.getName()) + .withFullyQualifiedName(testCase.getFullyQualifiedName()) + .withDescription(testCase.getDescription()) + .withDisplayName(testCase.getDisplayName()) + .withHref(testCase.getHref()) + .withDeleted(testCase.getDeleted())); + } + testSuite.setTests(testCasesEntityReferences); + return new RestUtil.PutResponse<>(Response.Status.OK, testSuite, LOGICAL_TEST_CASES_ADDED); } public RestUtil.DeleteResponse deleteTestCaseFromLogicalTestSuite(UUID testSuiteId, UUID testCaseId) throws IOException { TestCase testCase = Entity.getEntity(Entity.TEST_CASE, testCaseId, null, null); deleteRelationship(testSuiteId, TEST_SUITE, testCaseId, TEST_CASE, Relationship.CONTAINS); - return new RestUtil.DeleteResponse<>( - testCase, String.format(RestUtil.TEST_CASE_REMOVED_FROM_LOGICAL_TEST_SUITE, testSuiteId)); + EntityReference entityReference = Entity.getEntityReferenceById(TEST_SUITE, testSuiteId, Include.ALL); + testCase.setTestSuite(entityReference); + return new RestUtil.DeleteResponse<>(testCase, RestUtil.ENTITY_DELETED); + } + + public TestSummary getTestSummary() throws IOException { + List testCases = listAll(Fields.EMPTY_FIELDS, new ListFilter()); + List testCaseFQNs = testCases.stream().map(TestCase::getFullyQualifiedName).collect(Collectors.toList()); + + if (testCaseFQNs.isEmpty()) return new TestSummary(); + + List jsonList = + daoCollection.entityExtensionTimeSeriesDao().getLatestExtensionByFQNs(testCaseFQNs, TESTCASE_RESULT_EXTENSION); + + HashMap testCaseSummary = new HashMap<>(); + for (String json : jsonList) { + TestCaseResult testCaseResult = JsonUtils.readValue(json, TestCaseResult.class); + String status = testCaseResult.getTestCaseStatus().toString(); + testCaseSummary.put(status, testCaseSummary.getOrDefault(status, 0) + 1); + } + + return new TestSummary() + .withAborted(testCaseSummary.getOrDefault(TestCaseStatus.Aborted.toString(), 0)) + .withFailed(testCaseSummary.getOrDefault(TestCaseStatus.Failed.toString(), 0)) + .withSuccess(testCaseSummary.getOrDefault(TestCaseStatus.Success.toString(), 0)) + .withTotal(testCaseFQNs.size()); } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java index 92979084431..45b643f87a9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java @@ -3,18 +3,28 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.service.Entity.TEST_CASE; import static org.openmetadata.service.Entity.TEST_SUITE; +import static org.openmetadata.service.jdbi3.TestCaseRepository.TESTCASE_RESULT_EXTENSION; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.stream.Collectors; +import javax.ws.rs.core.SecurityContext; +import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.tests.TestSuite; +import org.openmetadata.schema.tests.type.TestCaseResult; +import org.openmetadata.schema.tests.type.TestCaseStatus; +import org.openmetadata.schema.tests.type.TestSummary; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; -import org.openmetadata.service.jdbi3.EntityRepository.EntityUpdater; import org.openmetadata.service.resources.dqtests.TestSuiteResource; import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.RestUtil; +@Slf4j public class TestSuiteRepository extends EntityRepository { private static final String UPDATE_FIELDS = "owner,tests"; private static final String PATCH_FIELDS = "owner,tests"; @@ -33,9 +43,33 @@ public class TestSuiteRepository extends EntityRepository { @Override public TestSuite setFields(TestSuite entity, EntityUtil.Fields fields) throws IOException { entity.setPipelines(fields.contains("pipelines") ? getIngestionPipelines(entity) : null); + entity.setSummary(fields.contains("summary") ? getTestSummary(entity) : null); return entity.withTests(fields.contains("tests") ? getTestCases(entity) : null); } + private TestSummary getTestSummary(TestSuite entity) throws IOException { + List testCases = getTestCases(entity); + HashMap testCaseSummary = new HashMap<>(); + List testCaseFQNs = + testCases.stream().map(EntityReference::getFullyQualifiedName).collect(Collectors.toList()); + + if (testCaseFQNs.isEmpty()) return new TestSummary(); + + List jsonList = + daoCollection.entityExtensionTimeSeriesDao().getLatestExtensionByFQNs(testCaseFQNs, TESTCASE_RESULT_EXTENSION); + + for (String json : jsonList) { + TestCaseResult testCaseResult = JsonUtils.readValue(json, TestCaseResult.class); + String status = testCaseResult.getTestCaseStatus().toString(); + testCaseSummary.put(status, testCaseSummary.getOrDefault(status, 0) + 1); + } + return new TestSummary() + .withAborted(testCaseSummary.getOrDefault(TestCaseStatus.Aborted.toString(), 0)) + .withFailed(testCaseSummary.getOrDefault(TestCaseStatus.Failed.toString(), 0)) + .withSuccess(testCaseSummary.getOrDefault(TestCaseStatus.Success.toString(), 0)) + .withTotal(testCaseFQNs.size()); + } + @Override public void prepare(TestSuite entity) { /* Nothing to do */ @@ -73,6 +107,34 @@ public class TestSuiteRepository extends EntityRepository { addRelationship(table.getId(), testSuite.getId(), Entity.TABLE, TEST_SUITE, Relationship.CONTAINS); } + public RestUtil.DeleteResponse deleteLogicalTestSuite( + SecurityContext securityContext, TestSuite original, boolean hardDelete) throws IOException { + // deleting a logical will delete the test suite and only remove + // the relationship to test cases if hardDelete is true. Test Cases + // will not be deleted. + String updatedBy = securityContext.getUserPrincipal().getName(); + preDelete(original); + setFieldsInternal(original, putFields); + + String changeType; + TestSuite updated = JsonUtils.readValue(JsonUtils.pojoToJson(original), TestSuite.class); + setFieldsInternal(updated, putFields); + + if (supportsSoftDelete && !hardDelete) { + updated.setUpdatedBy(updatedBy); + updated.setUpdatedAt(System.currentTimeMillis()); + updated.setDeleted(true); + EntityUpdater updater = getUpdater(original, updated, Operation.SOFT_DELETE); + updater.update(); + changeType = RestUtil.ENTITY_SOFT_DELETED; + } else { + cleanup(updated); + changeType = RestUtil.ENTITY_DELETED; + } + LOG.info("{} deleted {}", hardDelete ? "Hard" : "Soft", updated.getFullyQualifiedName()); + return new RestUtil.DeleteResponse<>(updated, changeType); + } + private EntityReference getIngestionPipeline(TestSuite testSuite) throws IOException { return getToEntityRef(testSuite.getId(), Relationship.CONTAINS, Entity.INGESTION_PIPELINE, false); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index fb10530c691..48edf7d854b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -27,6 +27,7 @@ import io.swagger.v3.oas.annotations.tags.Tag; import java.io.IOException; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; import javax.json.JsonPatch; import javax.validation.Valid; import javax.validation.constraints.Max; @@ -173,6 +174,13 @@ public class TableResource extends EntityResource { schema = @Schema(type = "string", example = "snowflakeWestCoast.financeDB.schema")) @QueryParam("databaseSchema") String databaseSchemaParam, + @Parameter( + description = + "Include tables with an empty test suite (i.e. no test cases have been created for this table). Default to true", + schema = @Schema(type = "boolean", example = "true")) + @QueryParam("includeEmptyTestSuite") + @DefaultValue("true") + boolean includeEmptyTestSuite, @Parameter(description = "Limit the number tables returned. (1 to 1000000, default = " + "10) ") @DefaultValue("10") @Min(0) @@ -196,7 +204,15 @@ public class TableResource extends EntityResource { new ListFilter(include) .addQueryParam("database", databaseParam) .addQueryParam("databaseSchema", databaseSchemaParam); - return super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after); + ResultList tableList = + super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after); + if (!includeEmptyTestSuite) { + tableList.setData( + tableList.getData().stream() + .filter(table -> table.getTestSuite() != null && !table.getTestSuite().getTests().isEmpty()) + .collect(Collectors.toList())); + } + return tableList; } @GET diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index 0fe06b8d622..e38bc43ae9d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -43,6 +43,7 @@ import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.tests.type.TestCaseResult; +import org.openmetadata.schema.tests.type.TestSummary; import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.EntityHistory; import org.openmetadata.schema.type.Include; @@ -682,7 +683,9 @@ public class TestCaseResource extends EntityResource testCaseIds = createLogicalTestCases.getTestCaseIds(); int existingTestCaseCount = repository.getTestCaseCount(testCaseIds); @@ -692,6 +695,26 @@ public class TestCaseResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/dataQuality/testSuites"; - static final String FIELDS = "owner,tests"; + public static final String EXECUTABLE_TEST_SUITE_DELETION_ERROR = + "Cannot delete logical test suite. To delete logical test suite, use DELETE /v1/dataQuality/testSuites/<...>"; + public static final String NON_EXECUTABLE_TEST_SUITE_DELETION_ERROR = + "Cannot delete executable test suite. To delete executable test suite, use DELETE /v1/dataQuality/testSuites/executable/<...>"; + + static final String FIELDS = "owner,tests,summary"; @Override public TestSuite addHref(UriInfo uriInfo, TestSuite testSuite) { @@ -113,6 +121,11 @@ public class TestSuiteResource extends EntityResource response = + repository.deleteLogicalTestSuite(securityContext, testSuite, hardDelete); + addHref(uriInfo, response.getEntity()); + return response.toResponse(); + } + @DELETE @Path("/name/{name}") + @Operation( + operationId = "deleteLogicalTestSuite", + summary = "Delete a logical test suite", + description = "Delete a logical test suite by `name`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "Logical Test suite for instance {name} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Hard delete the logical entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "FQN of the logical test suite", schema = @Schema(type = "String")) @PathParam("name") + String name) + throws IOException { + OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); + authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); + TestSuite testSuite = Entity.getEntityByName(Entity.TEST_SUITE, name, "*", ALL); + if (testSuite.getExecutable()) { + throw new IllegalArgumentException(NON_EXECUTABLE_TEST_SUITE_DELETION_ERROR); + } + RestUtil.DeleteResponse response = + repository.deleteLogicalTestSuite(securityContext, testSuite, hardDelete); + addHref(uriInfo, response.getEntity()); + return response.toResponse(); + } + + @DELETE + @Path("/executable/name/{name}") @Operation( operationId = "deleteTestSuiteByName", summary = "Delete a test suite", @@ -361,9 +439,13 @@ public class TestSuiteResource extends EntityResource response = + repository.deleteByName(securityContext.getUserPrincipal().getName(), name, recursive, hardDelete); + addHref(uriInfo, response.getEntity()); + return response.toResponse(); } @DELETE - @Path("/{id}") + @Path("/executable/{id}") @Operation( operationId = "deleteTestSuite", summary = "Delete a test suite", @@ -384,7 +475,7 @@ public class TestSuiteResource extends EntityResource response = + repository.delete(securityContext.getUserPrincipal().getName(), id, recursive, hardDelete); + addHref(uriInfo, response.getEntity()); + return response.toResponse(); } @PUT diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java index a5158a15eb6..142d501e27a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java @@ -258,6 +258,9 @@ public class SearchResource { case "query_search_index": searchSourceBuilder = buildQuerySearchBuilder(query, from, size); break; + case "test_case_search_index": + searchSourceBuilder = buildTestCaseSearch(query, from, size); + break; default: searchSourceBuilder = buildAggregateSearchBuilder(query, from, size); break; @@ -906,4 +909,37 @@ public class SearchResource { return searchBuilder(queryBuilder, hb, from, size); } + + private SearchSourceBuilder buildTestCaseSearch(String query, int from, int size) { + QueryStringQueryBuilder queryBuilder = + QueryBuilders.queryStringQuery(query) + .field(FIELD_NAME, 10.0f) + .field(DESCRIPTION, 3.0f) + .field("testSuite.fullyQualifiedName", 10.0f) + .field("testSuite.name", 10.0f) + .field("testSuite.description", 3.0f) + .field("entityLink", 3.0f) + .field("entityFQN", 10.0f) + .defaultOperator(Operator.AND) + .fuzziness(Fuzziness.AUTO); + + HighlightBuilder.Field highlightTestCaseDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION); + highlightTestCaseDescription.highlighterType(UNIFIED); + HighlightBuilder.Field highlightTestCaseName = new HighlightBuilder.Field(FIELD_NAME); + highlightTestCaseName.highlighterType(UNIFIED); + HighlightBuilder.Field highlightTestSuiteName = new HighlightBuilder.Field("testSuite.name"); + highlightTestSuiteName.highlighterType(UNIFIED); + HighlightBuilder.Field highlightTestSuiteDescription = new HighlightBuilder.Field("testSuite.description"); + highlightTestSuiteDescription.highlighterType(UNIFIED); + HighlightBuilder hb = new HighlightBuilder(); + hb.field(highlightTestCaseDescription); + hb.field(highlightTestCaseName); + hb.field(highlightTestSuiteName); + hb.field(highlightTestSuiteDescription); + + hb.preTags(PRE_TAG); + hb.postTags(POST_TAG); + + return searchBuilder(queryBuilder, hb, from, size); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/RestUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/RestUtil.java index 4b0d5b98c8e..0d05aebb2c2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/RestUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/RestUtil.java @@ -43,7 +43,7 @@ public final class RestUtil { public static final String DELETED_TEAM_NAME = "DeletedTeam"; public static final String DELETED_TEAM_DISPLAY = "Team was deleted"; public static final String SIGNATURE_HEADER = "X-OM-Signature"; - public static final String TEST_CASES_ADDED = "%s Test Cases Added to Test Suite %s"; + public static final String LOGICAL_TEST_CASES_ADDED = "Logical Test Cases Added to Test Suite"; public static final String TEST_CASE_REMOVED_FROM_LOGICAL_TEST_SUITE = "Test case successfuly removed from test suite ID %s"; @@ -146,7 +146,8 @@ public final class RestUtil { ResponseBuilder responseBuilder = Response.status(status).header(CHANGE_CUSTOM_HEADER, changeType); if (changeType.equals(RestUtil.ENTITY_CREATED) || changeType.equals(RestUtil.ENTITY_UPDATED) - || changeType.equals(RestUtil.ENTITY_NO_CHANGE)) { + || changeType.equals(RestUtil.ENTITY_NO_CHANGE) + || changeType.equals(RestUtil.LOGICAL_TEST_CASES_ADDED)) { return responseBuilder.entity(entity).build(); } else { return responseBuilder.entity(changeEvent).build(); diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/test_case_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/test_case_index_mapping.json new file mode 100644 index 00000000000..b3a6601fc9c --- /dev/null +++ b/openmetadata-service/src/main/resources/elasticsearch/en/test_case_index_mapping.json @@ -0,0 +1,274 @@ +{ + "settings": { + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, + "om_ngram": { + "tokenizer": "ngram", + "min_gram": 1, + "max_gram": 2, + "filter": [ + "lowercase" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + } + } + } + }, + "mappings": { + "properties": { + "id": { + "type": "text" + }, + "name": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "entityLink": { + "type": "text" + }, + "entityFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "parameterValues": { + "properties": { + "name": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "value": { + "type": "text" + } + } + }, + "testDefinition": { + "properties": { + "id": { + "type": "text" + }, + "name": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "entityType": { + "type": "keyword" + }, + "testPlatforms": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "owner": { + "properties": { + "id": { + "type": "text" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "testSuite": { + "properties": { + "id": { + "type": "text" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + }, + "executable": { + "type": "text" + } + } + }, + "version": { + "type": "float" + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "deleted": { + "type": "text" + } + } + } +} \ No newline at end of file diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index 3dbb4b70aa9..2124b899bc7 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -95,6 +95,8 @@ import org.openmetadata.schema.api.data.CreateQuery; import org.openmetadata.schema.api.data.CreateTable; import org.openmetadata.schema.api.data.CreateTableProfile; import org.openmetadata.schema.api.tests.CreateCustomMetric; +import org.openmetadata.schema.api.tests.CreateTestCase; +import org.openmetadata.schema.api.tests.CreateTestSuite; import org.openmetadata.schema.entity.data.Database; import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.entity.data.Query; @@ -102,6 +104,7 @@ import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.services.DatabaseService; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.tests.CustomMetric; +import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.type.Column; @@ -129,6 +132,8 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.databases.TableResource.TableList; +import org.openmetadata.service.resources.dqtests.TestCaseResourceTest; +import org.openmetadata.service.resources.dqtests.TestSuiteResourceTest; import org.openmetadata.service.resources.glossary.GlossaryResourceTest; import org.openmetadata.service.resources.glossary.GlossaryTermResourceTest; import org.openmetadata.service.resources.query.QueryResource; @@ -1288,7 +1293,7 @@ public class TableResourceTest extends EntityResourceTest { assertEquals(query1.getQuery(), createdQuery.getQuery()); assertEquals(query1.getDuration(), createdQuery.getDuration()); - // Update bote + // Update bot VoteRequest request = new VoteRequest().withUpdatedVoteType(VoteRequest.VoteType.VOTED_UP); WebTarget target = getResource(String.format("queries/%s/vote", createdQuery.getId().toString())); ChangeEvent changeEvent = TestUtils.put(target, request, ChangeEvent.class, OK, ADMIN_AUTH_HEADERS); @@ -1761,6 +1766,57 @@ public class TableResourceTest extends EntityResourceTest { assertEquals("P30D", table.getRetentionPeriod()); } + @Test + void get_tablesWithTestCases(TestInfo test) throws IOException { + TestCaseResourceTest testCaseResourceTest = new TestCaseResourceTest(); + TestSuiteResourceTest testSuiteResourceTest = new TestSuiteResourceTest(); + DatabaseSchemaResourceTest schemaResourceTest = new DatabaseSchemaResourceTest(); + DatabaseResourceTest databaseTest = new DatabaseResourceTest(); + + // Create Database + CreateDatabase createDatabase = databaseTest.createRequest(getEntityName(test)); + Database database = databaseTest.createEntity(createDatabase, ADMIN_AUTH_HEADERS); + // Create Database Schema + CreateDatabaseSchema createDatabaseSchema = + schemaResourceTest.createRequest(test).withDatabase(database.getFullyQualifiedName()); + DatabaseSchema schema = + schemaResourceTest + .createEntity(createDatabaseSchema, ADMIN_AUTH_HEADERS) + .withDatabase(database.getEntityReference()); + schema = schemaResourceTest.getEntity(schema.getId(), "", ADMIN_AUTH_HEADERS); + // Create Table 1 + CreateTable createTable1 = createRequest(test).withDatabaseSchema(schema.getFullyQualifiedName()); + Table table1 = createEntity(createTable1, ADMIN_AUTH_HEADERS).withDatabase(database.getEntityReference()); + // Create Table 2 + CreateTable createTable2 = + createRequest(test.getClass().getName() + "2").withDatabaseSchema(schema.getFullyQualifiedName()); + createEntity(createTable2, ADMIN_AUTH_HEADERS).withDatabase(database.getEntityReference()); + // Create Executable Test Suite + CreateTestSuite createExecutableTestSuite = testSuiteResourceTest.createRequest(table1.getFullyQualifiedName()); + TestSuite executableTestSuite = + testSuiteResourceTest.createExecutableTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); + + HashMap queryParams = new HashMap<>(); + queryParams.put("includeEmptyTestSuite", "false"); + queryParams.put("fields", "testSuite"); + queryParams.put("limit", "100"); + ResultList
tables = listEntities(queryParams, ADMIN_AUTH_HEADERS); + assertTrue(tables.getData().isEmpty()); + + for (int i = 0; i < 5; i++) { + CreateTestCase create = + testCaseResourceTest + .createRequest("test_testSuite__" + i) + .withTestSuite(executableTestSuite.getFullyQualifiedName()); + testCaseResourceTest.createAndCheckEntity(create, ADMIN_AUTH_HEADERS); + } + + tables = listEntities(queryParams, ADMIN_AUTH_HEADERS); + assertEquals(1, tables.getData().size()); + assertEquals(table1.getId(), tables.getData().get(0).getId()); + assertNotNull(tables.getData().get(0).getTestSuite()); + } + @Test void test_sensitivePIISampleData(TestInfo test) throws IOException { // Create table with owner and a column tagged with PII.Sensitive diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index 92137a4e193..9336a7cd99c 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -50,6 +50,7 @@ import org.openmetadata.schema.tests.type.TestCaseFailureStatus; import org.openmetadata.schema.tests.type.TestCaseFailureStatusType; import org.openmetadata.schema.tests.type.TestCaseResult; import org.openmetadata.schema.tests.type.TestCaseStatus; +import org.openmetadata.schema.tests.type.TestSummary; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.ColumnDataType; @@ -335,6 +336,11 @@ public class TestCaseResourceTest extends EntityResourceTest authHeaders) throws IOException { + WebTarget target = getCollection().path("/executionSummary"); + return TestUtils.get(target, TestSummary.class, authHeaders); + } + public ResultList getTestCases( Integer limit, String fields, String link, Boolean includeAll, Map authHeaders) throws HttpResponseException { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java index 0c6974cff5d..2fe6e84878d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java @@ -1,9 +1,9 @@ package org.openmetadata.service.resources.dqtests; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; -import static javax.ws.rs.core.Response.Status.CONFLICT; import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; import static org.openmetadata.service.util.TestUtils.LONG_ENTITY_NAME; @@ -107,7 +107,6 @@ public class TestSuiteResourceTest extends EntityResourceTest testCases1 = new ArrayList<>(); - List testCases2 = new ArrayList<>(); for (int i = 0; i < 5; i++) { CreateTestCase createTestCase = @@ -121,8 +120,7 @@ public class TestSuiteResourceTest extends EntityResourceTest actualTestSuites = getTestSuites(10, "*", ADMIN_AUTH_HEADERS); @@ -133,7 +131,7 @@ public class TestSuiteResourceTest extends EntityResourceTest getEntity(TEST_SUITE1.getId(), ADMIN_AUTH_HEADERS), NOT_FOUND, @@ -221,12 +219,12 @@ public class TestSuiteResourceTest extends EntityResourceTest testCaseId.getId()).collect(Collectors.toList())), - CONFLICT, - "Entity already exists"); + BAD_REQUEST, + "You are trying to add test cases to an executable test suite."); } @Test - void post_createExecTestSuiteNonExistingEntity_400(TestInfo test) throws IOException { + void post_createExecTestSuiteNonExistingEntity_400(TestInfo test) { CreateTestSuite createTestSuite = createRequest(test); assertResponse( () -> createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS), @@ -237,6 +235,7 @@ public class TestSuiteResourceTest extends EntityResourceTest getTestSuites(Integer limit, String fields, Map authHeaders) - throws HttpResponseException { - WebTarget target = getResource("dataQuality/testSuites"); - target = limit != null ? target.queryParam("limit", limit) : target; - target = target.queryParam("fields", fields); - return TestUtils.get(target, TestSuiteResource.TestSuiteList.class, authHeaders); + @Test + void get_filterTestSuiteType_200(TestInfo test) throws IOException { + // Create a logical test suite + CreateTestSuite createTestSuite = createRequest(test); + createEntity(createTestSuite, ADMIN_AUTH_HEADERS); + + Map queryParams = new HashMap<>(); + + ResultList testSuiteResultList = listEntities(queryParams, ADMIN_AUTH_HEADERS); + assertEquals(10, testSuiteResultList.getData().size()); + + queryParams.put("testSuiteType", "executable"); + testSuiteResultList = listEntities(queryParams, ADMIN_AUTH_HEADERS); + testSuiteResultList + .getData() + .forEach( + ts -> { + assertEquals(true, ts.getExecutable()); + }); + + queryParams.put("testSuiteType", "logical"); + testSuiteResultList = listEntities(queryParams, ADMIN_AUTH_HEADERS); + testSuiteResultList + .getData() + .forEach( + ts -> { + assertEquals(false, ts.getExecutable()); + }); } @Test @@ -333,6 +364,57 @@ public class TestSuiteResourceTest extends EntityResourceTest testCases = new ArrayList<>(); + + // We'll create tests cases for testSuite1 + for (int i = 0; i < 5; i++) { + CreateTestCase createTestCase = + testCaseResourceTest + .createRequest(String.format("test_testSuite_2_%s_", test.getDisplayName()) + i) + .withTestSuite(executableTestSuite.getFullyQualifiedName()); + TestCase testCase = testCaseResourceTest.createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS); + testCases.add(testCase.getEntityReference()); + } + + // We'll create a logical test suite and associate the test cases to it + CreateTestSuite createTestSuite = createRequest(test); + TestSuite testSuite = createEntity(createTestSuite, ADMIN_AUTH_HEADERS); + addTestCasesToLogicalTestSuite( + testSuite, testCases.stream().map(testCaseId -> testCaseId.getId()).collect(Collectors.toList())); + + // We'll delete the logical test suite + deleteEntity(testSuite.getId(), true, true, ADMIN_AUTH_HEADERS); + + // We'll check that the test cases are still present in the executable test suite + TestSuite actualExecutableTestSuite = getEntity(executableTestSuite.getId(), "*", ADMIN_AUTH_HEADERS); + assertEquals(actualExecutableTestSuite.getTests().size(), 5); + } + + public ResultList getTestSuites(Integer limit, String fields, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("dataQuality/testSuites"); + target = limit != null ? target.queryParam("limit", limit) : target; + target = target.queryParam("fields", fields); + return TestUtils.get(target, TestSuiteResource.TestSuiteList.class, authHeaders); + } + public TestSuite createExecutableTestSuite(CreateTestSuite createTestSuite, Map authHeaders) throws IOException { WebTarget target = getResource("dataQuality/testSuites/executable"); @@ -347,6 +429,14 @@ public class TestSuiteResourceTest extends EntityResourceTest authHeaders) + throws IOException { + WebTarget target = getResource(String.format("dataQuality/testSuites/executable/%s", id.toString())); + target = recursive ? target.queryParam("recursive", true) : target; + target = hardDelete ? target.queryParam("hardDelete", true) : target; + TestUtils.delete(target, TestSuite.class, authHeaders); + } + private void verifyTestSuites(ResultList actualTestSuites, List expectedTestSuites) { Map testSuiteMap = new HashMap<>(); for (TestSuite result : actualTestSuites.getData()) { @@ -356,6 +446,7 @@ public class TestSuiteResourceTest extends EntityResourceTest