From f9a3f9988e0751a4ae9d1ecc0b9b82acdb2e6029 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sat, 27 Aug 2022 00:31:24 -0700 Subject: [PATCH] =?UTF-8?q?Fix=20#6936:=20testPlatforms=20is=20required=20?= =?UTF-8?q?field=20in=20testSuite=20and=20createTes=E2=80=A6=20(#6944)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix #6936: testPlatforms is required field in testSuite and createTestSuite schema * Fix #6936: testPlatforms is required field in testSuite and createTestSuite schema * Fix #6936: testPlatforms is required field in testSuite and createTestSuite schema * Fix #6936: testPlatforms is required field in testSuite and createTestSuite schema * Fix #6936: testPlatforms is required field in testSuite and createTestSuite schema * fixed glossary test * Fixed sample data ingestion * Add filters for testCaseResultSuccess testCaseResultFailed, testCaseResultAborted * Fix #6936: fix testcase listing * Fix #6936: fix java style Co-authored-by: Shailesh Parmar Co-authored-by: Teddy Crepineau Co-authored-by: mohitdeuex --- .../v004__create_db_connection_info.sql | 13 ++- .../v005__create_db_connection_info.sql | 11 --- .../v004__create_db_connection_info.sql | 14 +++- .../v005__create_db_connection_info.sql | 11 --- .../catalog/jdbi3/CollectionDAO.java | 6 +- .../catalog/jdbi3/TestCaseRepository.java | 55 ++++++++++-- .../catalog/jdbi3/TestSuiteRepository.java | 9 ++ .../resources/dqtests/TestCaseResource.java | 10 +-- .../openmetadata/catalog/util/FilterUtil.java | 34 +++++++- .../schema/api/tests/createTestSuite.json | 6 +- .../json/schema/tests/testSuite.json | 2 +- .../dqtests/TestCaseResourceTest.java | 36 +++++--- .../dqtests/TestSuiteResourceTest.java | 84 ++++++++++++++++++- .../ingestion/source/database/sample_data.py | 1 - 14 files changed, 232 insertions(+), 60 deletions(-) delete mode 100644 bootstrap/sql/com.mysql.cj.jdbc.Driver/v005__create_db_connection_info.sql delete mode 100644 bootstrap/sql/org.postgresql.Driver/v005__create_db_connection_info.sql diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v004__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v004__create_db_connection_info.sql index 6aa7a2bddb1..27f5f81b476 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v004__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v004__create_db_connection_info.sql @@ -92,4 +92,15 @@ WHERE serviceType = 'Oracle'; UPDATE dbservice_entity SET json = JSON_REMOVE(json, '$.connection.config.hostPort') -WHERE serviceType = 'Athena'; \ No newline at end of file +WHERE serviceType = 'Athena'; +UPDATE dbservice_entity +SET json = JSON_REMOVE(json, '$.connection.config.username', '$.connection.config.password') +WHERE serviceType in ('Databricks'); + +CREATE TABLE IF NOT EXISTS openmetadata_settings ( + id MEDIUMINT NOT NULL AUTO_INCREMENT, + configType VARCHAR(36) NOT NULL, + json JSON NOT NULL, + PRIMARY KEY (id, configType), + UNIQUE(configType) +); diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v005__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v005__create_db_connection_info.sql deleted file mode 100644 index 19263f3178a..00000000000 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v005__create_db_connection_info.sql +++ /dev/null @@ -1,11 +0,0 @@ -UPDATE dbservice_entity -SET json = JSON_REMOVE(json, '$.connection.config.username', '$.connection.config.password') -WHERE serviceType in ('Databricks'); - -CREATE TABLE IF NOT EXISTS openmetadata_settings ( - id MEDIUMINT NOT NULL AUTO_INCREMENT, - configType VARCHAR(36) NOT NULL, - json JSON NOT NULL, - PRIMARY KEY (id, configType), - UNIQUE(configType) - ); diff --git a/bootstrap/sql/org.postgresql.Driver/v004__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v004__create_db_connection_info.sql index 30a36844426..14695197bc0 100644 --- a/bootstrap/sql/org.postgresql.Driver/v004__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v004__create_db_connection_info.sql @@ -87,4 +87,16 @@ WHERE serviceType = 'Oracle'; UPDATE dbservice_entity SET json = json::jsonb #- '{connection,config,hostPort}' -WHERE serviceType = 'Athena'; \ No newline at end of file +WHERE serviceType = 'Athena'; +UPDATE dbservice_entity +SET json = json::jsonb #- '{connection,config,username}' #- '{connection,config,password}' +WHERE serviceType in ('Databricks'); + +CREATE TABLE IF NOT EXISTS openmetadata_settings ( + id SERIAL NOT NULL , + configType VARCHAR(36) NOT NULL, + json JSONB NOT NULL, + PRIMARY KEY (id, configType), + UNIQUE(configType) +); + diff --git a/bootstrap/sql/org.postgresql.Driver/v005__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v005__create_db_connection_info.sql deleted file mode 100644 index 043364f0d67..00000000000 --- a/bootstrap/sql/org.postgresql.Driver/v005__create_db_connection_info.sql +++ /dev/null @@ -1,11 +0,0 @@ -UPDATE dbservice_entity -SET json = json::jsonb #- '{connection,config,username}' #- '{connection,config,password}' -WHERE serviceType in ('Databricks'); - -CREATE TABLE IF NOT EXISTS openmetadata_settings ( - id SERIAL NOT NULL , - configType VARCHAR(36) NOT NULL, - json JSONB NOT NULL, - PRIMARY KEY (id, configType), - UNIQUE(configType) - ); \ No newline at end of file diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java index 282f095f1bb..38c7d38c409 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java @@ -2560,7 +2560,7 @@ public interface CollectionDAO { condition = String.format( "%s AND id IN (SELECT toId FROM entity_relationship WHERE fromId='%s' AND toEntity='%s' AND relation=%d AND fromEntity='%s')", - condition, testSuiteId, Entity.TEST_CASE, Relationship.HAS.ordinal(), Entity.TEST_SUITE); + condition, testSuiteId, Entity.TEST_CASE, Relationship.CONTAINS.ordinal(), Entity.TEST_SUITE); } return listBefore(getTableName(), getNameColumn(), condition, limit, before); @@ -2587,7 +2587,7 @@ public interface CollectionDAO { condition = String.format( "%s AND id IN (SELECT toId FROM entity_relationship WHERE fromId='%s' AND toEntity='%s' AND relation=%d AND fromEntity='%s')", - condition, testSuiteId, Entity.TEST_CASE, Relationship.HAS.ordinal(), Entity.TEST_SUITE); + condition, testSuiteId, Entity.TEST_CASE, Relationship.CONTAINS.ordinal(), Entity.TEST_SUITE); } return listAfter(getTableName(), getNameColumn(), condition, limit, after); @@ -2614,7 +2614,7 @@ public interface CollectionDAO { condition = String.format( "%s AND id IN (SELECT toId FROM entity_relationship WHERE fromId='%s' AND toEntity='%s' AND relation=%d AND fromEntity='%s')", - condition, testSuiteId, Entity.TEST_CASE, Relationship.HAS.ordinal(), Entity.TEST_SUITE); + condition, testSuiteId, Entity.TEST_CASE, Relationship.CONTAINS.ordinal(), Entity.TEST_SUITE); } return listCount(getTableName(), getNameColumn(), condition); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TestCaseRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TestCaseRepository.java index 8fe737b94b7..63a7ea622ae 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TestCaseRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TestCaseRepository.java @@ -9,8 +9,10 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.ws.rs.core.Response; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.EntityInterface; import org.openmetadata.catalog.exception.EntityNotFoundException; import org.openmetadata.catalog.resources.dqtests.TestSuiteResource; import org.openmetadata.catalog.resources.feeds.MessageParser; @@ -19,7 +21,11 @@ import org.openmetadata.catalog.test.TestCaseParameterValue; import org.openmetadata.catalog.tests.TestCase; import org.openmetadata.catalog.tests.TestDefinition; import org.openmetadata.catalog.tests.type.TestCaseResult; +import org.openmetadata.catalog.type.ChangeDescription; +import org.openmetadata.catalog.type.ChangeEvent; import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.type.EventType; +import org.openmetadata.catalog.type.FieldChange; import org.openmetadata.catalog.type.Include; import org.openmetadata.catalog.type.Relationship; import org.openmetadata.catalog.util.EntityUtil; @@ -68,7 +74,7 @@ public class TestCaseRepository extends EntityRepository { } private EntityReference getTestSuite(TestCase test) throws IOException { - return getFromEntityRef(test.getId(), Relationship.HAS, null, true); + return getFromEntityRef(test.getId(), Relationship.CONTAINS, TEST_SUITE, true); } private EntityReference getTestDefinition(TestCase test) throws IOException { @@ -112,7 +118,7 @@ public class TestCaseRepository extends EntityRepository { MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse(test.getEntityLink()); EntityReference tableRef = EntityUtil.validateEntityLink(entityLink); // Add relationship from testSuite to test - addRelationship(test.getTestSuite().getId(), test.getId(), TEST_SUITE, TEST_CASE, Relationship.HAS); + addRelationship(test.getTestSuite().getId(), test.getId(), TEST_SUITE, TEST_CASE, Relationship.CONTAINS); // Add relationship from entity to test addRelationship(tableRef.getId(), test.getId(), tableRef.getType(), TEST_CASE, Relationship.CONTAINS); // Add relationship from test definition to test @@ -123,7 +129,7 @@ public class TestCaseRepository extends EntityRepository { } @Transaction - public TestCase addTestCaseResult(String fqn, TestCaseResult testCaseResult) throws IOException { + public RestUtil.PutResponse addTestCaseResult(String fqn, TestCaseResult testCaseResult) throws IOException { // Validate the request content TestCase testCase = dao.findEntityByName(fqn); @@ -152,11 +158,16 @@ public class TestCaseRepository extends EntityRepository { JsonUtils.pojoToJson(testCaseResult)); setFields(testCase, EntityUtil.Fields.EMPTY_FIELDS); } - return testCase.withTestCaseResult(testCaseResult); + + ChangeDescription change = + addTestCaseChangeDescription(testCase.getVersion(), testCaseResult, storedTestCaseResult); + ChangeEvent changeEvent = getChangeEvent(testCase, change, entityType, testCase.getVersion()); + + return new RestUtil.PutResponse<>(Response.Status.CREATED, changeEvent, RestUtil.ENTITY_FIELDS_CHANGED); } @Transaction - public TestCase deleteTestCaseResult(String fqn, Long timestamp) throws IOException { + public RestUtil.PutResponse deleteTestCaseResult(String fqn, Long timestamp) throws IOException { // Validate the request content TestCase testCase = dao.findEntityByName(fqn); TestCaseResult storedTestCaseResult = @@ -168,12 +179,44 @@ public class TestCaseRepository extends EntityRepository { if (storedTestCaseResult != null) { daoCollection.entityExtensionTimeSeriesDao().deleteAtTimestamp(fqn, TESTCASE_RESULT_EXTENSION, timestamp); testCase.setTestCaseResult(storedTestCaseResult); - return testCase; + ChangeDescription change = deleteTestCaseChangeDescription(testCase.getVersion(), storedTestCaseResult); + ChangeEvent changeEvent = getChangeEvent(testCase, change, entityType, testCase.getVersion()); + return new RestUtil.PutResponse<>(Response.Status.OK, changeEvent, RestUtil.ENTITY_FIELDS_CHANGED); } throw new EntityNotFoundException( String.format("Failed to find testCase result for %s at %s", testCase.getName(), timestamp)); } + private ChangeDescription addTestCaseChangeDescription(Double version, Object newValue, Object oldValue) { + FieldChange fieldChange = + new FieldChange().withName("testCaseResult").withNewValue(newValue).withOldValue(oldValue); + ChangeDescription change = new ChangeDescription().withPreviousVersion(version); + change.getFieldsUpdated().add(fieldChange); + return change; + } + + private ChangeDescription deleteTestCaseChangeDescription(Double version, Object oldValue) { + FieldChange fieldChange = new FieldChange().withName("testCaseResult").withOldValue(oldValue); + ChangeDescription change = new ChangeDescription().withPreviousVersion(version); + change.getFieldsDeleted().add(fieldChange); + return change; + } + + private ChangeEvent getChangeEvent( + EntityInterface updated, ChangeDescription change, String entityType, Double prevVersion) { + return new ChangeEvent() + .withEntity(updated) + .withChangeDescription(change) + .withEventType(EventType.ENTITY_UPDATED) + .withEntityType(entityType) + .withEntityId(updated.getId()) + .withEntityFullyQualifiedName(updated.getFullyQualifiedName()) + .withUserName(updated.getUpdatedBy()) + .withTimestamp(System.currentTimeMillis()) + .withCurrentVersion(updated.getVersion()) + .withPreviousVersion(prevVersion); + } + private TestCaseResult getTestCaseResult(TestCase testCase) throws IOException { return JsonUtils.readValue( daoCollection diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TestSuiteRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TestSuiteRepository.java index 4ff6a5bf3d8..be9fc7f2f32 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TestSuiteRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TestSuiteRepository.java @@ -1,8 +1,10 @@ package org.openmetadata.catalog.jdbi3; +import static org.openmetadata.catalog.Entity.TEST_CASE; import static org.openmetadata.catalog.Entity.TEST_SUITE; import java.io.IOException; +import java.util.List; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.resources.dqtests.TestSuiteResource; import org.openmetadata.catalog.tests.TestSuite; @@ -29,6 +31,7 @@ public class TestSuiteRepository extends EntityRepository { public TestSuite setFields(TestSuite entity, EntityUtil.Fields fields) throws IOException { entity.setOwner(fields.contains("owner") ? getOwner(entity) : null); entity.setPipeline(fields.contains("pipelines") ? getIngestionPipeline(entity) : null); + entity.setTests(fields.contains("tests") ? getTestCases(entity) : null); return entity; } @@ -37,6 +40,12 @@ public class TestSuiteRepository extends EntityRepository { setFullyQualifiedName(entity); } + private List getTestCases(TestSuite entity) throws IOException { + List testCases = + findTo(entity.getId(), TEST_SUITE, Relationship.CONTAINS, TEST_CASE); + return EntityUtil.getEntityReferences(testCases); + } + @Override public void storeEntity(TestSuite entity, boolean update) throws IOException { EntityReference owner = entity.getOwner(); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dqtests/TestCaseResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dqtests/TestCaseResource.java index 20300f0991d..86abfd2009e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dqtests/TestCaseResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dqtests/TestCaseResource.java @@ -381,15 +381,14 @@ public class TestCaseResource extends EntityResource> filtersMap) { if (filtersMap != null && !filtersMap.isEmpty()) { String entityType = changeEvent.getEntityType(); @@ -46,13 +50,35 @@ public class FilterUtil { if ((sf = filtersOfEntity.get(eventType)) == null) { return false; } else { - return sf.getFields().contains("all") || checkIfFilterContainField(sf, getUpdateField(changeEvent)); + if (sf.getFields().contains("all")) { + return true; + } else { + if (entityType.equals("testCase")) { + return handleTestCaseFilter(changeEvent, sf); + } else { + return checkIfFilterContainField(sf, getUpdateField(changeEvent)); + } + } } } } return false; } + private static boolean handleTestCaseFilter(ChangeEvent changeEvent, Filters sf) { + List fieldChanges = getAllFieldChange(changeEvent); + for (FieldChange fieldChange : fieldChanges) { + if (fieldChange.getName().equals(TEST_CASE_RESULT)) { + TestCaseResult testCaseResult = (TestCaseResult) fieldChange.getNewValue(); + TestCaseStatus status = testCaseResult.getTestCaseStatus(); + if (sf.getFields().contains(TEST_CASE_RESULT + status.toString())) { + return true; + } + } + } + return checkIfFilterContainField(sf, getUpdateField(changeEvent)); + } + public static boolean handleWithWildCardFilter( Map wildCardFilter, EventType type, List updatedField) { if (wildCardFilter != null && !wildCardFilter.isEmpty()) { @@ -81,13 +107,17 @@ public class FilterUtil { || changeEvent.getEventType() == EventType.ENTITY_SOFT_DELETED) { return List.of(changeEvent.getEntityType()); } + return getChangedFields(getAllFieldChange(changeEvent)); + } + + public static List getAllFieldChange(ChangeEvent changeEvent) { ChangeDescription description = changeEvent.getChangeDescription(); List allFieldChange = new ArrayList<>(); allFieldChange.addAll(description.getFieldsAdded()); allFieldChange.addAll(description.getFieldsUpdated()); allFieldChange.addAll(description.getFieldsDeleted()); - return getChangedFields(allFieldChange); + return allFieldChange; } public static List getChangedFields(List field) { diff --git a/catalog-rest-service/src/main/resources/json/schema/api/tests/createTestSuite.json b/catalog-rest-service/src/main/resources/json/schema/api/tests/createTestSuite.json index 93f31679aef..8fe2eadb5c0 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/tests/createTestSuite.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/tests/createTestSuite.json @@ -19,15 +19,11 @@ "description": "Description of the test suite.", "$ref": "../../type/basic.json#/definitions/markdown" }, - "scheduleInterval": { - "description": "TestSuite schedule interval in cron format.", - "type": "string" - }, "owner": { "description": "Owner of this test suite", "$ref": "../../type/entityReference.json" } }, - "required": ["name", "description", "testPlatforms"], + "required": ["name", "description"], "additionalProperties": false } diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/testSuite.json b/catalog-rest-service/src/main/resources/json/schema/tests/testSuite.json index bb8c3bf196f..6f91f220f2f 100644 --- a/catalog-rest-service/src/main/resources/json/schema/tests/testSuite.json +++ b/catalog-rest-service/src/main/resources/json/schema/tests/testSuite.json @@ -71,6 +71,6 @@ "default": false } }, - "required": ["name", "description", "testPlatform"], + "required": ["name", "description"], "additionalProperties": false } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dqtests/TestCaseResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dqtests/TestCaseResourceTest.java index 1c0ff2e76e1..7467191b4cf 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dqtests/TestCaseResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dqtests/TestCaseResourceTest.java @@ -1,6 +1,7 @@ package org.openmetadata.catalog.resources.dqtests; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.OK; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -35,6 +36,7 @@ import org.openmetadata.catalog.resources.EntityResourceTest; import org.openmetadata.catalog.resources.databases.TableResourceTest; import org.openmetadata.catalog.test.TestCaseParameterValue; import org.openmetadata.catalog.tests.TestCase; +import org.openmetadata.catalog.tests.TestSuite; import org.openmetadata.catalog.tests.type.TestCaseResult; import org.openmetadata.catalog.tests.type.TestCaseStatus; import org.openmetadata.catalog.type.ChangeDescription; @@ -202,8 +204,7 @@ public class TestCaseResourceTest extends EntityResourceTest testCaseResults = getTestCaseResults(testCase.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS); @@ -215,8 +216,7 @@ public class TestCaseResourceTest extends EntityResourceTest authHeaders) + public static void putTestCaseResult(String fqn, TestCaseResult data, Map authHeaders) throws HttpResponseException { WebTarget target = CatalogApplicationTest.getResource("testCase/" + fqn + "/testCaseResult"); - return TestUtils.put(target, data, TestCase.class, OK, authHeaders); + TestUtils.put(target, data, CREATED, authHeaders); } - public static TestCase deleteTestCaseResult(String fqn, Long timestamp, Map authHeaders) + public static void deleteTestCaseResult(String fqn, Long timestamp, Map authHeaders) throws HttpResponseException { WebTarget target = CatalogApplicationTest.getResource("testCase/" + fqn + "/testCaseResult/" + timestamp); - return TestUtils.delete(target, TestCase.class, authHeaders); + TestUtils.delete(target, authHeaders); } public static ResultList getTestCaseResults( @@ -373,6 +374,19 @@ public class TestCaseResourceTest extends EntityResourceTest getTestCases( + Integer limit, String fields, TestSuite testSuite, Boolean includeAll, Map authHeaders) + throws HttpResponseException { + WebTarget target = CatalogApplicationTest.getResource("testCase"); + target = limit != null ? target.queryParam("limit", limit) : target; + target = target.queryParam("fields", fields); + target = target.queryParam("testSuiteId", testSuite.getId()); + if (includeAll) { + target = target.queryParam("includeAllTests", true); + } + return TestUtils.get(target, TestCaseResource.TestCaseList.class, authHeaders); + } + private void verifyTestCaseResults( ResultList actualTestCaseResults, List expectedTestCaseResults, diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dqtests/TestSuiteResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dqtests/TestSuiteResourceTest.java index 5c4c1d27e67..7370b596685 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dqtests/TestSuiteResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dqtests/TestSuiteResourceTest.java @@ -6,14 +6,26 @@ import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS; import static org.openmetadata.catalog.util.TestUtils.assertResponse; import java.io.IOException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.UUID; +import javax.ws.rs.client.WebTarget; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.openmetadata.catalog.CatalogApplicationTest; import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.api.tests.CreateTestCase; import org.openmetadata.catalog.api.tests.CreateTestSuite; import org.openmetadata.catalog.resources.EntityResourceTest; +import org.openmetadata.catalog.tests.TestCase; import org.openmetadata.catalog.tests.TestSuite; +import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.util.ResultList; +import org.openmetadata.catalog.util.TestUtils; public class TestSuiteResourceTest extends EntityResourceTest { public TestSuiteResourceTest() { @@ -48,9 +60,79 @@ public class TestSuiteResourceTest extends EntityResourceTest testCases1 = new ArrayList<>(); + List testCases2 = new ArrayList<>(); + CreateTestSuite createTestSuite1 = createRequest(test); + TestSuite testSuite1 = createAndCheckEntity(createTestSuite1, ADMIN_AUTH_HEADERS); + CreateTestSuite createTestSuite2 = createRequest(test.getDisplayName() + UUID.randomUUID()); + TestSuite testSuite2 = createAndCheckEntity(createTestSuite2, ADMIN_AUTH_HEADERS); + + for (int i = 0; i < 5; i++) { + CreateTestCase createTestCase = + testCaseResourceTest.createRequest("test_testSuite_1_" + i).withTestSuite(testSuite1.getEntityReference()); + TestCase testCase = testCaseResourceTest.createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS); + testCases1.add(testCase.getEntityReference()); + } + + for (int i = 5; i < 10; i++) { + CreateTestCase create = + testCaseResourceTest.createRequest("test_testSuite_2_" + i).withTestSuite(testSuite2.getEntityReference()); + TestCase testCase = testCaseResourceTest.createAndCheckEntity(create, ADMIN_AUTH_HEADERS); + testCases2.add(testCase.getEntityReference()); + } + + ResultList actualTestSuites = getTestSuites(10, "*", ADMIN_AUTH_HEADERS); + verifyTestSuites(actualTestSuites, List.of(createTestSuite1, createTestSuite2)); + + for (TestSuite testSuite : actualTestSuites.getData()) { + if (testSuite.getName().equals(createTestSuite1.getName())) { + verifyTestCases(testSuite.getTests(), testCases1); + } + } + } + + public static ResultList getTestSuites(Integer limit, String fields, Map authHeaders) + throws HttpResponseException { + WebTarget target = CatalogApplicationTest.getResource("testSuite"); + target = limit != null ? target.queryParam("limit", limit) : target; + target = target.queryParam("fields", fields); + return TestUtils.get(target, TestSuiteResource.TestSuiteList.class, authHeaders); + } + + private void verifyTestSuites(ResultList actualTestSuites, List expectedTestSuites) + throws HttpResponseException { + Map testSuiteMap = new HashMap<>(); + for (TestSuite result : actualTestSuites.getData()) { + testSuiteMap.put(result.getName(), result); + } + for (CreateTestSuite result : expectedTestSuites) { + TestSuite storedTestSuite = testSuiteMap.get(result.getName()); + if (storedTestSuite == null) continue; + validateCreatedEntity(storedTestSuite, result, ADMIN_AUTH_HEADERS); + } + } + + private void verifyTestCases(List actualTestCases, List expectedTestCases) + throws HttpResponseException { + assertEquals(expectedTestCases.size(), actualTestCases.size()); + Map testCaseMap = new HashMap<>(); + for (EntityReference result : actualTestCases) { + testCaseMap.put(result.getId(), result); + } + for (EntityReference result : expectedTestCases) { + EntityReference storedTestCase = testCaseMap.get(result.getId()); + assertEquals(result.getId(), storedTestCase.getId()); + assertEquals(result.getName(), storedTestCase.getName()); + assertEquals(result.getDescription(), storedTestCase.getDescription()); + } + } + @Override public CreateTestSuite createRequest(String name) { - return new CreateTestSuite().withName(name).withDescription(name).withScheduleInterval("* * 1 * *"); + return new CreateTestSuite().withName(name).withDescription(name); } @Override diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index 73b67ef50fd..954d9e0143c 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -867,7 +867,6 @@ class SampleDataSource(Source[Entity]): test_suite=CreateTestSuiteRequest( name=test_suite["testSuiteName"], description=test_suite["testSuiteDescription"], - scheduleInterval=test_suite["scheduleInterval"], ) )