From 030d576df7e309ec56f589afd666b043dadbf0fd Mon Sep 17 00:00:00 2001 From: Teddy Date: Wed, 19 Oct 2022 16:30:46 +0200 Subject: [PATCH] Fix test suite bug for redefined quotes (#8248) * Fixed bug when running test for same table name across different DB * Fix python check style * Fixed linting --- .../interfaces/sqalchemy/sqa_test_suite_interface.py | 5 ++++- ingestion/src/metadata/test_suite/api/workflow.py | 9 +++++++-- .../content/deployment/upgrade/versions/011-to-012.md | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ingestion/src/metadata/interfaces/sqalchemy/sqa_test_suite_interface.py b/ingestion/src/metadata/interfaces/sqalchemy/sqa_test_suite_interface.py index 31fe7e6ddbf..c57cc98cd3a 100644 --- a/ingestion/src/metadata/interfaces/sqalchemy/sqa_test_suite_interface.py +++ b/ingestion/src/metadata/interfaces/sqalchemy/sqa_test_suite_interface.py @@ -17,6 +17,7 @@ supporting sqlalchemy abstraction layer from datetime import datetime, timezone from typing import Optional, Union +from sqlalchemy import MetaData from sqlalchemy.orm import DeclarativeMeta from sqlalchemy.orm.util import AliasedClass @@ -45,10 +46,12 @@ class SQATestSuiteInterface(SQAInterfaceMixin, TestSuiteProtocol): against a SQAlchemy source. """ + # pylint: disable=too-many-arguments def __init__( self, service_connection_config: DatabaseConnection, ometa_client: OpenMetadata, + sqa_metadata_obj: Optional[MetaData] = None, table_sample_precentage: float = None, table_sample_query: str = None, table_partition_config: dict = None, @@ -62,7 +65,7 @@ class SQATestSuiteInterface(SQAInterfaceMixin, TestSuiteProtocol): ) self.set_session_tag(self.session) - self._table = self._convert_table_to_orm_object() + self._table = self._convert_table_to_orm_object(sqa_metadata_obj) self.table_sample_precentage = table_sample_precentage self.table_sample_query = table_sample_query diff --git a/ingestion/src/metadata/test_suite/api/workflow.py b/ingestion/src/metadata/test_suite/api/workflow.py index 7c242913aa5..4c941703a42 100644 --- a/ingestion/src/metadata/test_suite/api/workflow.py +++ b/ingestion/src/metadata/test_suite/api/workflow.py @@ -22,6 +22,7 @@ from logging import Logger from typing import List, Optional, Set, Tuple from pydantic import ValidationError +from sqlalchemy import MetaData from metadata.config.common import WorkflowExecutionError from metadata.config.workflow import get_sink @@ -236,7 +237,7 @@ class TestSuiteWorkflow: ) return None - def _create_runner_interface(self, entity_fqn: str): + def _create_runner_interface(self, entity_fqn: str, sqa_metadata_obj: MetaData): """create the interface to execute test against SQA sources""" table_entity = self._get_table_entity_from_test_case(entity_fqn) return SQATestSuiteInterface( @@ -244,6 +245,7 @@ class TestSuiteWorkflow: entity_fqn ), ometa_client=create_ometa_client(self.metadata_config), + sqa_metadata_obj=sqa_metadata_obj, table_entity=table_entity, table_sample_precentage=self._get_profile_sample(table_entity) if not self._get_profile_query(table_entity) @@ -412,8 +414,11 @@ class TestSuiteWorkflow: unique_entity_fqns = self._get_unique_entities_from_test_cases(test_cases) for entity_fqn in unique_entity_fqns: + sqa_metadata_obj = MetaData() try: - runner_interface = self._create_runner_interface(entity_fqn) + runner_interface = self._create_runner_interface( + entity_fqn, sqa_metadata_obj + ) data_test_runner = self._create_data_tests_runner(runner_interface) for test_case in self._filter_test_cases_for_entity( diff --git a/openmetadata-docs/content/deployment/upgrade/versions/011-to-012.md b/openmetadata-docs/content/deployment/upgrade/versions/011-to-012.md index f082048f2da..077721209e2 100644 --- a/openmetadata-docs/content/deployment/upgrade/versions/011-to-012.md +++ b/openmetadata-docs/content/deployment/upgrade/versions/011-to-012.md @@ -67,7 +67,7 @@ In the `processor` you can now configure: In OpenMetadata 0.12 we have migrated the metrics computation to multithreading. This migration reduced metrics computation time by 70%. -For Snowflake users, there is a known issue with the python package `snowflake-connector-python` in Python 3.9 where multithreading creates a circular import of the package. We highly recommend to either 1) run the ingestion workflow in Python 3.8 environment or 2) if you can't manage your environement set `ThreadCount` to 1. You can find more information on the profiler setting [here](/connectors/ingestion/workflows/profiler) +Snowflake users may experience a circular import error. This is a known issue with `snowflake-connector-python`. If you experience such error we recommend to either 1) run the ingestion workflow in Python 3.8 environment or 2) if you can't manage your environement set `ThreadCount` to 1. You can find more information on the profiler setting [here](/connectors/ingestion/workflows/profiler) ### Airflow Version