From 06735fe8dbaac5b267c9a2cf744ca154f88a9247 Mon Sep 17 00:00:00 2001 From: Teddy Date: Tue, 6 Jun 2023 09:45:49 +0200 Subject: [PATCH] Fixe Issue #11863 - Add Status logic for test case results (#11881) * feat: added entityReference field in testSuite to link testSuite to an entity when the testSuite is executable. * feat: added `executableEntityReference` as an entity reference for executable test suite to their entity * feat: add status object to test case results * feat: ran python linting --- .../sample_data/tests/testSuites.json | 3 +- .../src/metadata/data_quality/api/workflow.py | 66 +++++++---- .../validations/base_test_handler.py | 15 +++ .../ingestion/source/database/sample_data.py | 1 + .../ometa/test_ometa_test_suite.py | 8 +- .../integration/test_suite/test_workflow.py | 28 +++-- .../test_suite/test_validations_databases.py | 12 +- .../test_suite/test_validations_datalake.py | 20 +++- .../service/jdbi3/TestCaseRepository.java | 27 +++++ .../service/jdbi3/TestSuiteRepository.java | 4 +- .../resources/dqtests/TestCaseResource.java | 34 ++++++ .../resources/dqtests/TestSuiteResource.java | 29 +++-- .../dqtests/TestCaseResourceTest.java | 103 ++++++++++++++++++ .../dqtests/TestSuiteResourceTest.java | 1 + .../schema/api/tests/createTestSuite.json | 6 +- .../metadataIngestion/testSuitePipeline.json | 2 +- .../resources/json/schema/tests/basic.json | 60 ++++++++++ .../json/schema/tests/testSuite.json | 4 + 18 files changed, 377 insertions(+), 46 deletions(-) diff --git a/ingestion/examples/sample_data/tests/testSuites.json b/ingestion/examples/sample_data/tests/testSuites.json index c1fc47a2ff2..89b949a834d 100644 --- a/ingestion/examples/sample_data/tests/testSuites.json +++ b/ingestion/examples/sample_data/tests/testSuites.json @@ -1,7 +1,8 @@ { "tests": [ { - "testSuiteName": "sample_data.ecommerce_db.shopify.dim_address", + "testSuiteName": "sample_data.ecommerce_db.shopify.dim_address.TestSuite", + "executableEntityReference": "sample_data.ecommerce_db.shopify.dim_address", "testSuiteDescription": "This is an executable test suite linked to an entity", "scheduleInterval": "0 0 * * MON", "testCases": [ diff --git a/ingestion/src/metadata/data_quality/api/workflow.py b/ingestion/src/metadata/data_quality/api/workflow.py index 290b696217c..8d6ec7bc8e1 100644 --- a/ingestion/src/metadata/data_quality/api/workflow.py +++ b/ingestion/src/metadata/data_quality/api/workflow.py @@ -50,6 +50,7 @@ from metadata.generated.schema.metadataIngestion.workflow import ( OpenMetadataWorkflowConfig, ) from metadata.generated.schema.tests.testCase import TestCase +from metadata.generated.schema.tests.testDefinition import TestDefinition, TestPlatform from metadata.generated.schema.tests.testSuite import TestSuite from metadata.generated.schema.type.basic import EntityLink, FullyQualifiedEntityName from metadata.ingestion.api.parser import parse_workflow_config_gracefully @@ -115,6 +116,10 @@ class TestSuiteWorkflow(WorkflowStatusMixin): self.status = ProcessorStatus() + self.table_entity: Optional[Table] = self._get_table_entity( + self.source_config.entityFullyQualifiedName.__root__ + ) + if self.config.sink: self.sink = get_sink( sink_type=self.config.sink.type, @@ -142,7 +147,7 @@ class TestSuiteWorkflow(WorkflowStatusMixin): ) raise err - def _get_table_entity(self, entity_fqn: str): + def _get_table_entity(self, entity_fqn: str) -> Optional[Table]: """given an entity fqn return the table entity Args: @@ -151,20 +156,17 @@ class TestSuiteWorkflow(WorkflowStatusMixin): return self.metadata.get_by_name( entity=Table, fqn=entity_fqn, - fields=["tableProfilerConfig"], + fields=["tableProfilerConfig", "testSuite"], ) - def get_test_suite_entity(self) -> Optional[TestSuite]: + def create_or_return_test_suite_entity(self) -> Optional[TestSuite]: """ try to get test suite name from source.servicName. In the UI workflow we'll write the entity name (i.e. the test suite) to source.serviceName. """ - test_suite = self.metadata.get_by_name( - entity=TestSuite, - fqn=self.source_config.entityFullyQualifiedName.__root__, - ) - + self.table_entity = cast(Table, self.table_entity) # satisfy type checker + test_suite = self.table_entity.testSuite if test_suite and not test_suite.executable: logger.debug( f"Test suite {test_suite.fullyQualifiedName.__root__} is not executable." @@ -180,10 +182,11 @@ class TestSuiteWorkflow(WorkflowStatusMixin): ) test_suite = self.metadata.create_or_update_executable_test_suite( CreateTestSuiteRequest( - name=self.source_config.entityFullyQualifiedName.__root__, - displayName=self.source_config.entityFullyQualifiedName.__root__, + name=f"{self.source_config.entityFullyQualifiedName.__root__}.TestSuite", + displayName=f"{self.source_config.entityFullyQualifiedName.__root__} Test Suite", description="Test Suite created from YAML processor config file", owner=None, + executableEntityReference=self.source_config.entityFullyQualifiedName.__root__, ) ) @@ -209,10 +212,33 @@ class TestSuiteWorkflow(WorkflowStatusMixin): cli_test_cases = cast( List[TestCaseDefinition], cli_test_cases ) # satisfy type checker - test_cases = self.compare_and_create_test_cases(cli_test_cases, test_cases) + test_cases = self.compare_and_create_test_cases( + cli_test_cases, test_cases, test_suite + ) return test_cases + def filter_for_om_test_cases(self, test_cases: List[TestCase]) -> List[TestCase]: + """ + Filter test cases for OM test cases only. This will prevent us from running non OM test cases + + Args: + test_cases: list of test cases + """ + om_test_cases: List[TestCase] = [] + for test_case in test_cases: + test_definition: TestDefinition = self.metadata.get_by_id( + TestDefinition, test_case.testDefinition.id + ) + if TestPlatform.OpenMetadata not in test_definition.testPlatforms: + logger.debug( + f"Test case {test_case.name.__root__} is not an OpenMetadata test case." + ) + continue + om_test_cases.append(test_case) + + return om_test_cases + def get_test_case_from_cli_config( self, ) -> Optional[List[TestCaseDefinition]]: @@ -257,6 +283,7 @@ class TestSuiteWorkflow(WorkflowStatusMixin): self, cli_test_cases_definitions: Optional[List[TestCaseDefinition]], test_cases: List[TestCase], + test_suite: TestSuite, ) -> Optional[List[TestCase]]: """ compare test cases defined in CLI config workflow with test cases @@ -306,7 +333,7 @@ class TestSuiteWorkflow(WorkflowStatusMixin): test_case_to_create.columnName, ) ), - testSuite=self.source_config.entityFullyQualifiedName, + testSuite=test_suite.fullyQualifiedName.__root__, parameterValues=list(test_case_to_create.parameterValues) if test_case_to_create.parameterValues else None, @@ -330,17 +357,14 @@ class TestSuiteWorkflow(WorkflowStatusMixin): def run_test_suite(self): """Main logic to run the tests""" - table_entity: Table = self._get_table_entity( - self.source_config.entityFullyQualifiedName.__root__ - ) - if not table_entity: + if not self.table_entity: logger.debug(traceback.format_exc()) raise ValueError( - f"Could not retrieve table entity for {self.source_config.entityFullyQualifiedName.__root__}" + f"Could not retrieve table entity for {self.source_config.entityFullyQualifiedName.__root__}. " "Make sure the table exists in OpenMetadata and/or the JWT Token provided is valid." ) - test_suite = self.get_test_suite_entity() + test_suite = self.create_or_return_test_suite_entity() if not test_suite: logger.debug( f"No test suite found for table {self.source_config.entityFullyQualifiedName.__root__} " @@ -356,14 +380,16 @@ class TestSuiteWorkflow(WorkflowStatusMixin): ) return + openmetadata_test_cases = self.filter_for_om_test_cases(test_cases) + test_suite_runner = test_suite_source_factory.create( self.config.source.type.lower(), self.config, self.metadata, - table_entity, + self.table_entity, ).get_data_quality_runner() - for test_case in test_cases: + for test_case in openmetadata_test_cases: try: test_result = test_suite_runner.run_and_handle(test_case) if not test_result: diff --git a/ingestion/src/metadata/data_quality/validations/base_test_handler.py b/ingestion/src/metadata/data_quality/validations/base_test_handler.py index 3e423f55b6a..7318e363e87 100644 --- a/ingestion/src/metadata/data_quality/validations/base_test_handler.py +++ b/ingestion/src/metadata/data_quality/validations/base_test_handler.py @@ -21,11 +21,14 @@ from datetime import datetime from typing import Callable, List, Optional, TypeVar, Union from metadata.generated.schema.tests.basic import ( + TestCaseFailureStatus, + TestCaseFailureStatusType, TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue +from metadata.generated.schema.type.basic import Timestamp from metadata.profiler.processor.runner import QueryRunner T = TypeVar("T", bound=Callable) @@ -102,12 +105,24 @@ class BaseTestValidator(ABC): Returns: TestCaseResult: """ + if status == TestCaseStatus.Failed: + test_case_failure_status = TestCaseFailureStatus( + testCaseFailureStatusType=TestCaseFailureStatusType.New, + testCaseFailureReason=None, + testCaseFailureComment=None, + updatedAt=Timestamp(__root__=int(datetime.utcnow().timestamp() * 1000)), + updatedBy=None, + ) + else: + test_case_failure_status = None + return TestCaseResult( timestamp=execution_date, # type: ignore testCaseStatus=status, result=result, testResultValue=test_result_value, sampleData=None, + testCaseFailureStatus=test_case_failure_status, ) def format_column_list(self, status: TestCaseStatus, cols: List): diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index 2f8c14f4aa7..14eaa6974c9 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -1098,6 +1098,7 @@ class SampleDataSource( test_suite=CreateTestSuiteRequest( name=test_suite["testSuiteName"], description=test_suite["testSuiteDescription"], + executableEntityReference=test_suite["executableEntityReference"], ) ) diff --git a/ingestion/tests/integration/ometa/test_ometa_test_suite.py b/ingestion/tests/integration/ometa/test_ometa_test_suite.py index 10568faa899..62529c65d44 100644 --- a/ingestion/tests/integration/ometa/test_ometa_test_suite.py +++ b/ingestion/tests/integration/ometa/test_ometa_test_suite.py @@ -82,8 +82,9 @@ class OMetaTestSuiteTest(TestCase): cls.test_suite: TestSuite = cls.metadata.create_or_update_executable_test_suite( CreateTestSuiteRequest( - name="sample_data.ecommerce_db.shopify.dim_address", + name="sample_data.ecommerce_db.shopify.dim_address.TestSuite", description="This is a test suite for the integration tests", + executableEntityReference="sample_data.ecommerce_db.shopify.dim_address", ) ) @@ -111,10 +112,11 @@ class OMetaTestSuiteTest(TestCase): def test_get_or_create_test_suite(self): """test we get a test suite object""" test_suite = self.metadata.get_or_create_test_suite( - "sample_data.ecommerce_db.shopify.dim_address" + "sample_data.ecommerce_db.shopify.dim_address.TestSuite" ) assert ( - test_suite.name.__root__ == "sample_data.ecommerce_db.shopify.dim_address" + test_suite.name.__root__ + == "sample_data.ecommerce_db.shopify.dim_address.TestSuite" ) assert isinstance(test_suite, TestSuite) diff --git a/ingestion/tests/integration/test_suite/test_workflow.py b/ingestion/tests/integration/test_suite/test_workflow.py index e756902ac7b..1bef851ece6 100644 --- a/ingestion/tests/integration/test_suite/test_workflow.py +++ b/ingestion/tests/integration/test_suite/test_workflow.py @@ -23,6 +23,7 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata OpenMetadataConnection, ) from metadata.generated.schema.tests.testCase import TestCase +from metadata.generated.schema.tests.testDefinition import TestDefinition from metadata.generated.schema.tests.testSuite import TestSuite from metadata.ingestion.ometa.ometa_api import OpenMetadata @@ -115,21 +116,22 @@ class TestSuiteWorkflowTests(unittest.TestCase): _test_suite_config.update(processor) workflow = TestSuiteWorkflow.create(_test_suite_config) - workflow_test_suite = workflow.get_test_suite_entity() + workflow_test_suite = workflow.create_or_return_test_suite_entity() test_suite = self.metadata.get_by_name( - entity=TestSuite, fqn="sample_data.ecommerce_db.shopify.dim_address" + entity=TestSuite, + fqn="sample_data.ecommerce_db.shopify.dim_address.TestSuite", ) assert workflow_test_suite.id == test_suite.id self.test_suite_ids = [test_suite.id] - def test_get_test_suite_entity(self): + def test_create_or_return_test_suite_entity(self): """test we can correctly retrieve a test suite""" _test_suite_config = deepcopy(test_suite_config) workflow = TestSuiteWorkflow.create(_test_suite_config) - test_suite = workflow.get_test_suite_entity() + test_suite = workflow.create_or_return_test_suite_entity() expected_test_suite = self.metadata.get_by_name( entity=TestSuite, fqn="critical_metrics_suite" @@ -162,7 +164,7 @@ class TestSuiteWorkflowTests(unittest.TestCase): _test_suite_config.update(processor) workflow = TestSuiteWorkflow.create(_test_suite_config) - test_suite = workflow.get_test_suite_entity() + test_suite = workflow.create_or_return_test_suite_entity() test_cases = workflow.get_test_cases_from_test_suite(test_suite) assert isinstance(test_cases, MutableSequence) @@ -258,7 +260,7 @@ class TestSuiteWorkflowTests(unittest.TestCase): fqn="sample_data.ecommerce_db.shopify.dim_address.address_id.my_test_case_two", ) - test_suite = workflow.get_test_suite_entity() + test_suite = workflow.create_or_return_test_suite_entity() test_cases = self.metadata.list_entities( entity=TestCase, fields=["testSuite", "entityLink", "testDefinition"], @@ -266,7 +268,7 @@ class TestSuiteWorkflowTests(unittest.TestCase): ).entities config_test_cases_def = workflow.get_test_case_from_cli_config() created_test_case = workflow.compare_and_create_test_cases( - config_test_cases_def, test_cases + config_test_cases_def, test_cases, test_suite ) # clean up test @@ -297,3 +299,15 @@ class TestSuiteWorkflowTests(unittest.TestCase): ) assert isinstance(service_connection, Table) + + # def test_filter_for_om_test_cases(self): + # """test filter for OM test cases method""" + # om_test_case_1 = TestCase( + # name="om_test_case_1", + # testDefinition=self.metadata.get_entity_reference( + # TestDefinition, + # "columnValuesToMatchRegex" + # ), + # entityLink="", + # testSuite=self.metadata.get_entity_reference("sample_data.ecommerce_db.shopify.dim_address.TestSuite"), + # ) diff --git a/ingestion/tests/unit/test_suite/test_validations_databases.py b/ingestion/tests/unit/test_suite/test_validations_databases.py index cd631b54875..105742ab168 100644 --- a/ingestion/tests/unit/test_suite/test_validations_databases.py +++ b/ingestion/tests/unit/test_suite/test_validations_databases.py @@ -20,7 +20,11 @@ from unittest.mock import patch import pytest from metadata.data_quality.validations.validator import Validator -from metadata.generated.schema.tests.basic import TestCaseResult, TestCaseStatus +from metadata.generated.schema.tests.basic import ( + TestCaseFailureStatusType, + TestCaseResult, + TestCaseStatus, +) from metadata.utils.importer import import_test_case_class EXECUTION_DATE = datetime.strptime("2021-07-03", "%Y-%m-%d") @@ -335,3 +339,9 @@ def test_suite_validation_database( if val_2: assert res.testResultValue[1].value == val_2 assert res.testCaseStatus == status + if res.testCaseStatus == TestCaseStatus.Failed: + assert ( + res.testCaseFailureStatus.testCaseFailureStatusType + == TestCaseFailureStatusType.New + ) + assert res.testCaseFailureStatus.updatedAt is not None diff --git a/ingestion/tests/unit/test_suite/test_validations_datalake.py b/ingestion/tests/unit/test_suite/test_validations_datalake.py index 3a9ed5747f3..35359d1d34f 100644 --- a/ingestion/tests/unit/test_suite/test_validations_datalake.py +++ b/ingestion/tests/unit/test_suite/test_validations_datalake.py @@ -21,7 +21,11 @@ import pytest from pandas import DataFrame from metadata.data_quality.validations.validator import Validator -from metadata.generated.schema.tests.basic import TestCaseResult, TestCaseStatus +from metadata.generated.schema.tests.basic import ( + TestCaseFailureStatusType, + TestCaseResult, + TestCaseStatus, +) from metadata.utils.importer import import_test_case_class EXECUTION_DATE = datetime.strptime("2021-07-03", "%Y-%m-%d") @@ -148,7 +152,7 @@ DATALAKE_DATA_FRAME = lambda times_increase_sample_data: DataFrame( "test_case_column_values_missing_count_to_be_equal", "columnValuesMissingCount", "COLUMN", - (TestCaseResult, "2000", None, TestCaseStatus.Success), + (TestCaseResult, "2000", None, TestCaseStatus.Failed), ), ( "test_case_column_values_missing_count_to_be_equal_missing_values", @@ -249,7 +253,7 @@ DATALAKE_DATA_FRAME = lambda times_increase_sample_data: DataFrame( "test_case_table_row_count_to_be_between", "tableRowCountToBeBetween", "TABLE", - (TestCaseResult, "6000", None, TestCaseStatus.Success), + (TestCaseResult, "6000", None, TestCaseStatus.Failed), ), ( "test_case_table_row_count_to_be_equal", @@ -298,4 +302,12 @@ def test_suite_validation_datalake( assert res.testResultValue[0].value == val_1 if val_2: assert res.testResultValue[1].value == val_2 - assert res.testCaseStatus == status + + assert res.testCaseStatus == status + + if res.testCaseStatus == TestCaseStatus.Failed: + assert ( + res.testCaseFailureStatus.testCaseFailureStatusType + == TestCaseFailureStatusType.New + ) + assert res.testCaseFailureStatus.updatedAt is not None diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java index 77b0172141b..444b9f4f001 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java @@ -3,12 +3,16 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.service.Entity.TEST_CASE; import static org.openmetadata.service.Entity.TEST_DEFINITION; import static org.openmetadata.service.Entity.TEST_SUITE; +import static org.openmetadata.service.util.RestUtil.ENTITY_NO_CHANGE; +import static org.openmetadata.service.util.RestUtil.ENTITY_UPDATED; import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.UUID; +import javax.json.JsonPatch; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import org.jdbi.v3.sqlobject.transaction.Transaction; @@ -55,6 +59,29 @@ public class TestCaseRepository extends EntityRepository { return test.withTestCaseResult(fields.contains(TEST_CASE_RESULT_FIELD) ? getTestCaseResult(test) : null); } + public RestUtil.PatchResponse patchTestCaseResults( + String fqn, Long timestamp, UriInfo uriInfo, String user, JsonPatch patch) throws IOException { + String change = ENTITY_NO_CHANGE; + TestCaseResult original = + JsonUtils.readValue( + daoCollection + .entityExtensionTimeSeriesDao() + .getExtensionAtTimestamp(fqn, TESTCASE_RESULT_EXTENSION, timestamp), + TestCaseResult.class); + + TestCaseResult updated = JsonUtils.applyPatch(original, patch, TestCaseResult.class); + + if (!Objects.equals(original.getTestCaseFailureStatus(), updated.getTestCaseFailureStatus())) { + updated.getTestCaseFailureStatus().setUpdatedBy(user); + updated.getTestCaseFailureStatus().setUpdatedAt(System.currentTimeMillis()); + daoCollection + .entityExtensionTimeSeriesDao() + .update(fqn, TESTCASE_RESULT_EXTENSION, JsonUtils.pojoToJson(updated), timestamp); + change = ENTITY_UPDATED; + } + return new RestUtil.PatchResponse<>(Response.Status.OK, updated, change); + } + @Override public void setFullyQualifiedName(TestCase test) { EntityLink entityLink = EntityLink.parse(test.getEntityLink()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java index 0718e6c18c1..92979084431 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java @@ -67,7 +67,9 @@ public class TestSuiteRepository extends EntityRepository { } public void storeExecutableRelationship(TestSuite testSuite) throws IOException { - Table table = Entity.getEntityByName(Entity.TABLE, testSuite.getName(), null, null); + Table table = + Entity.getEntityByName( + Entity.TABLE, testSuite.getExecutableEntityReference().getFullyQualifiedName(), null, null); addRelationship(table.getId(), testSuite.getId(), Entity.TABLE, TEST_SUITE, Relationship.CONTAINS); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index 46cf91325d9..9b6b23d911b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -361,6 +361,40 @@ public class TestCaseResource extends EntityResource patchResponse = + dao.patchTestCaseResults(fqn, timestamp, uriInfo, securityContext.getUserPrincipal().getName(), patch); + return patchResponse.toResponse(); + } + @PUT @Operation( operationId = "createOrUpdateTest", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java index d3d51e99bae..9e81476d503 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java @@ -34,8 +34,10 @@ import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.api.tests.CreateTestSuite; +import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; @@ -248,6 +250,7 @@ public class TestSuiteResource extends EntityResource testCaseResultResultList = + getTestCaseResults( + testCase.getFullyQualifiedName(), + TestUtils.dateToTimestamp("2021-09-09"), + TestUtils.dateToTimestamp("2021-09-09"), + ADMIN_AUTH_HEADERS); + + assertEquals( + testCaseResultResultList.getData().get(0).getTestCaseFailureStatus().getTestCaseFailureStatusType(), + TestCaseFailureStatusType.New); + + String original = JsonUtils.pojoToJson(testCaseResult); + testCaseResult + .getTestCaseFailureStatus() + .withTestCaseFailureStatusType(TestCaseFailureStatusType.Resolved) + .withTestCaseFailureReason(TestCaseFailureReason.FalsePositive) + .withTestCaseFailureComment("Test failure was a false positive"); + + JsonPatch patch = JsonUtils.getJsonPatch(original, JsonUtils.pojoToJson(testCaseResult)); + + patchTestCaseResult(testCase.getFullyQualifiedName(), dateToTimestamp("2021-09-09"), patch, ADMIN_AUTH_HEADERS); + + ResultList testCaseResultResultListUpdated = + getTestCaseResults( + testCase.getFullyQualifiedName(), + TestUtils.dateToTimestamp("2021-09-09"), + TestUtils.dateToTimestamp("2021-09-09"), + ADMIN_AUTH_HEADERS); + + assertEquals( + testCaseResultResultListUpdated.getData().get(0).getTestCaseFailureStatus().getTestCaseFailureStatusType(), + TestCaseFailureStatusType.Resolved); + assertEquals( + testCaseResultResultListUpdated.getData().get(0).getTestCaseFailureStatus().getTestCaseFailureReason(), + TestCaseFailureReason.FalsePositive); + } + + @Test + void patch_testCaseResults_noChange(TestInfo test) throws IOException, ParseException { + CreateTestCase create = + createRequest(test) + .withEntityLink(TABLE_LINK_2) + .withTestSuite(TEST_SUITE1.getFullyQualifiedName()) + .withTestDefinition(TEST_DEFINITION3.getFullyQualifiedName()) + .withParameterValues(List.of(new TestCaseParameterValue().withValue("100").withName("missingCountValue"))); + TestCase testCase = createAndCheckEntity(create, ADMIN_AUTH_HEADERS); + + TestCaseResult testCaseResult = + new TestCaseResult() + .withResult("tested") + .withTestCaseStatus(TestCaseStatus.Success) + .withTimestamp(TestUtils.dateToTimestamp("2021-09-09")); + putTestCaseResult(testCase.getFullyQualifiedName(), testCaseResult, ADMIN_AUTH_HEADERS); + + String original = JsonUtils.pojoToJson(testCaseResult); + testCaseResult.setTestCaseStatus(TestCaseStatus.Failed); + JsonPatch patch = JsonUtils.getJsonPatch(original, JsonUtils.pojoToJson(testCaseResult)); + + patchTestCaseResult(testCase.getFullyQualifiedName(), dateToTimestamp("2021-09-09"), patch, ADMIN_AUTH_HEADERS); + + ResultList testCaseResultResultListUpdated = + getTestCaseResults( + testCase.getFullyQualifiedName(), + TestUtils.dateToTimestamp("2021-09-09"), + TestUtils.dateToTimestamp("2021-09-09"), + ADMIN_AUTH_HEADERS); + + // patching anything else than the test case failure status should not change anything + assertEquals(testCaseResultResultListUpdated.getData().get(0).getTestCaseStatus(), TestCaseStatus.Success); + } + @Test @Override public void delete_entity_as_non_admin_401(TestInfo test) throws HttpResponseException { @@ -572,6 +668,13 @@ public class TestCaseResourceTest extends EntityResourceTest authHeaders) + throws HttpResponseException { + WebTarget target = getCollection().path("/" + testCaseFqn + "/testCaseResult/" + timestamp); + return TestUtils.patch(target, patch, TestCaseResult.class, authHeaders); + } + private void verifyTestCaseResults( ResultList actualTestCaseResults, List expectedTestCaseResults, diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java index 92dfc672d2e..0c6974cff5d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java @@ -336,6 +336,7 @@ public class TestSuiteResourceTest extends EntityResourceTest authHeaders) throws IOException { WebTarget target = getResource("dataQuality/testSuites/executable"); + createTestSuite.setExecutableEntityReference(createTestSuite.getName()); return TestUtils.post(target, createTestSuite, TestSuite.class, authHeaders); } diff --git a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestSuite.json b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestSuite.json index 30f18c5b33d..8029542ab99 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestSuite.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestSuite.json @@ -16,7 +16,7 @@ }, "properties": { "name": { - "description": "Name that identifies this test suite. For executable testSuite, this should match the an entity FQN in the platform.", + "description": "Name that identifies this test suite.", "$ref": "#/definitions/testSuiteEntityName" }, "displayName": { @@ -30,6 +30,10 @@ "owner": { "description": "Owner of this test suite", "$ref": "../../type/entityReference.json" + }, + "executableEntityReference": { + "description": "FQN of the entity the test suite is executed against.. Only applicable for executable test suites.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" } }, "required": ["name"], diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/testSuitePipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/testSuitePipeline.json index 65775fdfed2..ee1b756fc31 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/testSuitePipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/testSuitePipeline.json @@ -18,7 +18,7 @@ "default": "TestSuite" }, "entityFullyQualifiedName": { - "description": "Name of the test suite. For executable test suite it should match a fully qualified name of the table", + "description": "Fully qualified name of the entity to be tested.", "$ref": "../type/basic.json#/definitions/fullyQualifiedEntityName" }, "profileSample": { diff --git a/openmetadata-spec/src/main/resources/json/schema/tests/basic.json b/openmetadata-spec/src/main/resources/json/schema/tests/basic.json index a2b1dac781b..b2152a46a6d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/tests/basic.json +++ b/openmetadata-spec/src/main/resources/json/schema/tests/basic.json @@ -62,6 +62,66 @@ "items": { "$ref": "#/definitions/testResultValue" } + }, + "testCaseFailureStatus": { + "description": "Schema to capture test case result.", + "javaType": "org.openmetadata.schema.tests.type.TestCaseFailureStatus", + "type": "object", + "properties": + { + "testCaseFailureStatusType": { + "description": "Status of Test Case Acknowledgement.", + "javaType": "org.openmetadata.schema.tests.type.TestCaseFailureStatusType", + "type": "string", + "enum": ["Ack", "New", "Resolved"], + "javaEnums": [ + { + "name": "Ack" + }, + { + "name": "New" + }, + { + "name": "Resolved" + } + ] + }, + "testCaseFailureReason": { + "description": "Reason of Test Case resolution.", + "javaType": "org.openmetadata.schema.tests.type.TestCaseFailureReason", + "type": "string", + "enum": ["FalsePositive", "MissingData", "Duplicates", "OutOfBounds", "Other"], + "javaEnums": [ + { + "name": "FalsePositive" + }, + { + "name": "MissingData" + }, + { + "name": "Duplicates" + }, + { + "name": "OutOfBounds" + }, + { + "name": "Other" + } + ] + }, + "testCaseFailureComment": { + "description": "Test case failure resolution comment.", + "type": "string" + }, + "updatedBy": { + "description": "User who updated the test case failure status.", + "type": "string" + }, + "updatedAt": { + "description": "Time when test case failure status was updated.", + "$ref": "../type/basic.json#/definitions/timestamp" + } + } } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json b/openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json index 70dccd22a47..bc334bc5048 100644 --- a/openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json +++ b/openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json @@ -101,6 +101,10 @@ "description": "Indicates if the test suite is executable. Set on the backend.", "type": "boolean", "default": false + }, + "executableEntityReference": { + "description": "Entity reference the test suite is executed against. Only applicable if the test suite is executable.", + "$ref": "../type/entityReference.json" } }, "required": ["name", "description"],