diff --git a/bootstrap/sql/migrations/native/1.4.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.4.0/mysql/schemaChanges.sql index 94ca33aa463..dc1b65a392d 100644 --- a/bootstrap/sql/migrations/native/1.4.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.4.0/mysql/schemaChanges.sql @@ -280,4 +280,6 @@ SET json = JSON_INSERT( JSON_REMOVE(json, '$.viewDefinition'), '$.schemaDefinition', JSON_EXTRACT(json, '$.viewDefinition') -); \ No newline at end of file +); + +UPDATE table_entity SET json = JSON_REMOVE(json, '$.testSuite'); diff --git a/bootstrap/sql/migrations/native/1.4.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.4.0/postgres/schemaChanges.sql index 3a42eed387d..1022f704cf5 100644 --- a/bootstrap/sql/migrations/native/1.4.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.4.0/postgres/schemaChanges.sql @@ -277,4 +277,7 @@ SET json = jsonb_set( '{schemaDefinition}', json->'viewDefinition' ) - 'viewDefinition' -WHERE jsonb_exists(json::jsonb, 'viewDefinition') = true; \ No newline at end of file +WHERE jsonb_exists(json::jsonb, 'viewDefinition') = true; + +UPDATE table_entity SET json = json - 'testSuite'; + diff --git a/ingestion/src/metadata/data_quality/source/test_suite.py b/ingestion/src/metadata/data_quality/source/test_suite.py index 7ea8056edc0..bea4a02e218 100644 --- a/ingestion/src/metadata/data_quality/source/test_suite.py +++ b/ingestion/src/metadata/data_quality/source/test_suite.py @@ -139,7 +139,13 @@ class TestSuiteSource(Source): ) ) - if table.testSuite and not table.testSuite.executable: + test_suite: Optional[TestSuite] = None + if table.testSuite: + test_suite = self.metadata.get_by_id( + entity=TestSuite, entity_id=table.testSuite.id.__root__ + ) + + if test_suite and not test_suite.executable: yield Either( left=StackTraceError( name="Non-executable Test Suite", @@ -149,7 +155,7 @@ class TestSuiteSource(Source): ) else: - test_suite_cases = self._get_test_cases_from_test_suite(table.testSuite) + test_suite_cases = self._get_test_cases_from_test_suite(test_suite) yield Either( right=TableAndTests( diff --git a/ingestion/tests/integration/great_expectations/test_great_expectation_integration.py b/ingestion/tests/integration/great_expectations/test_great_expectation_integration.py index c94ed50f2e0..6d5e24a1717 100644 --- a/ingestion/tests/integration/great_expectations/test_great_expectation_integration.py +++ b/ingestion/tests/integration/great_expectations/test_great_expectation_integration.py @@ -30,6 +30,7 @@ from metadata.generated.schema.entity.services.databaseService import DatabaseSe from metadata.generated.schema.security.client.openMetadataJWTClientConfig import ( OpenMetadataJWTClientConfig, ) +from metadata.generated.schema.tests.testSuite import TestSuite from metadata.ingestion.connections.session import create_and_bind_session from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.utils.time_utils import ( @@ -217,7 +218,10 @@ class TestGreatExpectationIntegration(TestCase): ) assert table_entity.testSuite - assert len(table_entity.testSuite.tests) == 1 + test_suite: TestSuite = self.metadata.get_by_id( + entity=TestSuite, entity_id=table_entity.testSuite.id, fields=["tests"] + ) + assert len(test_suite.tests) == 1 test_case_results = self.metadata.get_test_case_results( test_case_fqn=TEST_CASE_FQN, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 335654df455..af657bf40b4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -28,7 +28,7 @@ import static org.openmetadata.service.Entity.DATABASE_SCHEMA; import static org.openmetadata.service.Entity.FIELD_OWNER; import static org.openmetadata.service.Entity.FIELD_TAGS; import static org.openmetadata.service.Entity.TABLE; -import static org.openmetadata.service.Entity.getEntity; +import static org.openmetadata.service.Entity.TEST_SUITE; import static org.openmetadata.service.Entity.populateEntityFieldTags; import static org.openmetadata.service.util.EntityUtil.getLocalColumnName; import static org.openmetadata.service.util.FullyQualifiedName.getColumnName; @@ -64,7 +64,6 @@ import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.feed.Suggestion; import org.openmetadata.schema.tests.CustomMetric; -import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.ColumnDataType; import org.openmetadata.schema.type.ColumnJoin; @@ -324,22 +323,8 @@ public class TableRepository extends EntityRepository { TableProfilerConfig.class); } - public TestSuite getTestSuite(Table table) { - List entityRelationshipRecords = - daoCollection - .relationshipDAO() - .findTo(table.getId(), TABLE, Relationship.CONTAINS.ordinal()); - Optional testSuiteRelationshipRecord = - entityRelationshipRecords.stream() - .filter( - entityRelationshipRecord -> - entityRelationshipRecord.getType().equals(Entity.TEST_SUITE)) - .findFirst(); - return testSuiteRelationshipRecord - .map( - entityRelationshipRecord -> - getEntity(Entity.TEST_SUITE, entityRelationshipRecord.getId(), "*", Include.ALL)) - .orElse(null); + public EntityReference getTestSuite(Table table) { + return getToEntityRef(table.getId(), Relationship.CONTAINS, TEST_SUITE, false); } @Transaction diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java index 6f4302f0392..f623e8efd5d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java @@ -475,6 +475,7 @@ public class TestSuiteResourceTest extends EntityResourceTest queryParams = new HashMap<>(); + queryParams.put("include", Include.ALL.value()); + Table actualTable = tableResourceTest.getEntity(table.getId(), "testSuite", ADMIN_AUTH_HEADERS); - TestSuite tableTestSuite = actualTable.getTestSuite(); - assertEquals(testSuite.getId(), tableTestSuite.getId()); + EntityReference tableTestSuiteRef = actualTable.getTestSuite(); + assertEquals(testSuite.getId(), tableTestSuiteRef.getId()); + TestSuite tableTestSuite = + testSuiteResourceTest.getEntity( + tableTestSuiteRef.getId(), queryParams, "tests", ADMIN_AUTH_HEADERS); assertEquals(5, tableTestSuite.getTests().size()); // Soft delete entity deleteExecutableTestSuite(tableTestSuite.getId(), true, false, ADMIN_AUTH_HEADERS); - actualTable = tableResourceTest.getEntity(actualTable.getId(), "testSuite", ADMIN_AUTH_HEADERS); - tableTestSuite = actualTable.getTestSuite(); + actualTable = + tableResourceTest.getEntity( + actualTable.getId(), queryParams, "testSuite", ADMIN_AUTH_HEADERS); + tableTestSuiteRef = actualTable.getTestSuite(); + tableTestSuite = + testSuiteResourceTest.getEntity(tableTestSuiteRef.getId(), "tests", ADMIN_AUTH_HEADERS); assertEquals(true, tableTestSuite.getDeleted()); // Hard delete entity @@ -518,6 +529,8 @@ public class TestSuiteResourceTest extends EntityResourceTest queryParams = new HashMap<>(); + queryParams.put("include", Include.ALL.value()); + Table actualTable = tableResourceTest.getEntity(table.getId(), "testSuite", ADMIN_AUTH_HEADERS); - TestSuite actualTestSuite = actualTable.getTestSuite(); + EntityReference actualTestSuiteRef = actualTable.getTestSuite(); + TestSuite actualTestSuite = + testSuiteResourceTest.getEntity( + actualTestSuiteRef.getId(), queryParams, "tests", ADMIN_AUTH_HEADERS); assertEquals(actualTestSuite.getId(), testSuite.getId()); tableResourceTest.deleteEntity(actualTable.getId(), true, false, ADMIN_AUTH_HEADERS); - HashMap queryParams = new HashMap<>(); - queryParams.put("include", Include.ALL.value()); + actualTestSuite = getEntityByName(testSuite.getFullyQualifiedName(), queryParams, "*", ADMIN_AUTH_HEADERS); assertEquals(true, actualTestSuite.getDeleted()); diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json index 0132a65222f..d6ad347ff23 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json @@ -1039,7 +1039,7 @@ }, "testSuite": { "description": "Executable test suite associated with this table", - "$ref": "../../tests/testSuite.json" + "$ref": "../../type/entityReference.json" }, "dataModel": { "description": "This captures information about how the table is modeled. Currently only DBT model is supported.",