Fix test suite bug for redefined quotes (#8248)

* Fixed bug when running test for same table name across different DB

* Fix python check style

* Fixed linting
This commit is contained in:
Teddy 2022-10-19 16:30:46 +02:00 committed by GitHub
parent d9b50b5be9
commit 030d576df7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 12 additions and 4 deletions

View File

@ -17,6 +17,7 @@ supporting sqlalchemy abstraction layer
from datetime import datetime, timezone
from typing import Optional, Union
from sqlalchemy import MetaData
from sqlalchemy.orm import DeclarativeMeta
from sqlalchemy.orm.util import AliasedClass
@ -45,10 +46,12 @@ class SQATestSuiteInterface(SQAInterfaceMixin, TestSuiteProtocol):
against a SQAlchemy source.
"""
# pylint: disable=too-many-arguments
def __init__(
self,
service_connection_config: DatabaseConnection,
ometa_client: OpenMetadata,
sqa_metadata_obj: Optional[MetaData] = None,
table_sample_precentage: float = None,
table_sample_query: str = None,
table_partition_config: dict = None,
@ -62,7 +65,7 @@ class SQATestSuiteInterface(SQAInterfaceMixin, TestSuiteProtocol):
)
self.set_session_tag(self.session)
self._table = self._convert_table_to_orm_object()
self._table = self._convert_table_to_orm_object(sqa_metadata_obj)
self.table_sample_precentage = table_sample_precentage
self.table_sample_query = table_sample_query

View File

@ -22,6 +22,7 @@ from logging import Logger
from typing import List, Optional, Set, Tuple
from pydantic import ValidationError
from sqlalchemy import MetaData
from metadata.config.common import WorkflowExecutionError
from metadata.config.workflow import get_sink
@ -236,7 +237,7 @@ class TestSuiteWorkflow:
)
return None
def _create_runner_interface(self, entity_fqn: str):
def _create_runner_interface(self, entity_fqn: str, sqa_metadata_obj: MetaData):
"""create the interface to execute test against SQA sources"""
table_entity = self._get_table_entity_from_test_case(entity_fqn)
return SQATestSuiteInterface(
@ -244,6 +245,7 @@ class TestSuiteWorkflow:
entity_fqn
),
ometa_client=create_ometa_client(self.metadata_config),
sqa_metadata_obj=sqa_metadata_obj,
table_entity=table_entity,
table_sample_precentage=self._get_profile_sample(table_entity)
if not self._get_profile_query(table_entity)
@ -412,8 +414,11 @@ class TestSuiteWorkflow:
unique_entity_fqns = self._get_unique_entities_from_test_cases(test_cases)
for entity_fqn in unique_entity_fqns:
sqa_metadata_obj = MetaData()
try:
runner_interface = self._create_runner_interface(entity_fqn)
runner_interface = self._create_runner_interface(
entity_fqn, sqa_metadata_obj
)
data_test_runner = self._create_data_tests_runner(runner_interface)
for test_case in self._filter_test_cases_for_entity(

View File

@ -67,7 +67,7 @@ In the `processor` you can now configure:
In OpenMetadata 0.12 we have migrated the metrics computation to multithreading. This migration reduced metrics computation time by 70%.
For Snowflake users, there is a known issue with the python package `snowflake-connector-python` in Python 3.9 where multithreading creates a circular import of the package. We highly recommend to either 1) run the ingestion workflow in Python 3.8 environment or 2) if you can't manage your environement set `ThreadCount` to 1. You can find more information on the profiler setting [here](/connectors/ingestion/workflows/profiler)
Snowflake users may experience a circular import error. This is a known issue with `snowflake-connector-python`. If you experience such error we recommend to either 1) run the ingestion workflow in Python 3.8 environment or 2) if you can't manage your environement set `ThreadCount` to 1. You can find more information on the profiler setting [here](/connectors/ingestion/workflows/profiler)
### Airflow Version