fixed #9656 - Add support for date type to column values to be between (#10890)

* fix: renamed  to  submodule

* fix: linting

* fix: columnValuesToBeBetween test for date column type
This commit is contained in:
Teddy 2023-04-04 17:16:44 +02:00 committed by GitHub
parent e6ac7af49d
commit 9b4e9132ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
138 changed files with 410 additions and 250 deletions

View File

@ -16,12 +16,12 @@ import os
import yaml
from metadata.data_insight.api.workflow import DataInsightWorkflow
from metadata.data_quality.api.workflow import TestSuiteWorkflow
from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
PipelineType,
)
from metadata.ingestion.api.workflow import Workflow
from metadata.profiler.api.workflow import ProfilerWorkflow
from metadata.test_suite.api.workflow import TestSuiteWorkflow
WORKFLOW_MAP = {
PipelineType.metadata.value: Workflow,

View File

@ -17,7 +17,7 @@ import sys
import traceback
from metadata.config.common import load_config_file
from metadata.test_suite.api.workflow import TestSuiteWorkflow
from metadata.data_quality.api.workflow import TestSuiteWorkflow
from metadata.utils.logger import cli_logger
from metadata.utils.workflow_output_handler import WorkflowType, print_init_error

View File

@ -25,6 +25,18 @@ from pydantic import BaseModel, ValidationError
from sqlalchemy import MetaData
from metadata.config.common import WorkflowExecutionError
from metadata.data_quality.api.models import (
TestCaseDefinition,
TestSuiteDefinition,
TestSuiteProcessorConfig,
)
from metadata.data_quality.interface.pandas.pandas_test_suite_interface import (
PandasTestSuiteInterface,
)
from metadata.data_quality.interface.sqlalchemy.sqa_test_suite_interface import (
SQATestSuiteInterface,
)
from metadata.data_quality.runner.core import DataTestsRunner
from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest
from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest
from metadata.generated.schema.entity.data.table import PartitionProfilerConfig, Table
@ -53,17 +65,7 @@ from metadata.ingestion.ometa.client_utils import create_ometa_client
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.connections import get_connection
from metadata.ingestion.source.database.datalake.metadata import ometa_to_dataframe
from metadata.interfaces.datalake.datalake_test_suite_interface import (
DataLakeTestSuiteInterface,
)
from metadata.interfaces.sqalchemy.sqa_test_suite_interface import SQATestSuiteInterface
from metadata.profiler.api.models import ProfileSampleConfig
from metadata.test_suite.api.models import (
TestCaseDefinition,
TestSuiteDefinition,
TestSuiteProcessorConfig,
)
from metadata.test_suite.runner.core import DataTestsRunner
from metadata.utils import entity_link
from metadata.utils.importer import get_sink
from metadata.utils.logger import test_suite_logger
@ -127,7 +129,7 @@ class TestSuiteWorkflow(WorkflowStatusMixin):
sink_type=self.config.sink.type,
sink_config=self.config.sink,
metadata_config=self.metadata_config,
from_="test_suite",
from_="data_quality",
)
@classmethod
@ -283,7 +285,7 @@ class TestSuiteWorkflow(WorkflowStatusMixin):
table_sample_query=table_sample_query,
table_partition_config=table_partition_config,
)
return DataLakeTestSuiteInterface(
return PandasTestSuiteInterface(
service_connection_config=service_connection_config,
ometa_client=self.client,
df=ometa_to_dataframe(

View File

@ -16,6 +16,8 @@ supporting sqlalchemy abstraction layer
from datetime import datetime, timezone
from typing import Optional
from metadata.data_quality.interface.test_suite_protocol import TestSuiteProtocol
from metadata.data_quality.validations.validator import Validator
from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
DatalakeConnection,
)
@ -23,16 +25,14 @@ from metadata.generated.schema.tests.basic import TestCaseResult
from metadata.generated.schema.tests.testCase import TestCase
from metadata.generated.schema.tests.testDefinition import TestDefinition
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.interfaces.datalake.mixins.pandas_mixin import PandasInterfaceMixin
from metadata.interfaces.test_suite_protocol import TestSuiteProtocol
from metadata.test_suite.validations.validator import Validator
from metadata.mixins.pandas.pandas_mixin import PandasInterfaceMixin
from metadata.utils.importer import import_test_case_class
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()
class DataLakeTestSuiteInterface(TestSuiteProtocol, PandasInterfaceMixin):
class PandasTestSuiteInterface(TestSuiteProtocol, PandasInterfaceMixin):
"""
Sequential interface protocol for testSuite and Profiler. This class
implements specific operations needed to run profiler and test suite workflow

View File

@ -21,6 +21,8 @@ from sqlalchemy import MetaData
from sqlalchemy.orm import DeclarativeMeta
from sqlalchemy.orm.util import AliasedClass
from metadata.data_quality.interface.test_suite_protocol import TestSuiteProtocol
from metadata.data_quality.validations.validator import Validator
from metadata.generated.schema.entity.data.table import PartitionProfilerConfig, Table
from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
from metadata.generated.schema.tests.basic import TestCaseResult
@ -29,12 +31,10 @@ from metadata.generated.schema.tests.testDefinition import TestDefinition
from metadata.ingestion.connections.session import create_and_bind_session
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.connections import get_connection
from metadata.interfaces.sqalchemy.mixins.sqa_mixin import SQAInterfaceMixin
from metadata.interfaces.test_suite_protocol import TestSuiteProtocol
from metadata.mixins.sqalchemy.sqa_mixin import SQAInterfaceMixin
from metadata.profiler.api.models import ProfileSampleConfig
from metadata.profiler.profiler.runner import QueryRunner
from metadata.profiler.profiler.sampler import Sampler
from metadata.test_suite.validations.validator import Validator
from metadata.profiler.processor.runner import QueryRunner
from metadata.profiler.processor.sampler import Sampler
from metadata.utils.constants import TEN_MIN
from metadata.utils.importer import import_test_case_class
from metadata.utils.logger import test_suite_logger

View File

@ -14,9 +14,9 @@ Main class to run data tests
"""
from metadata.data_quality.interface.test_suite_protocol import TestSuiteProtocol
from metadata.data_quality.runner.models import TestCaseResultResponse
from metadata.generated.schema.tests.testCase import TestCase
from metadata.interfaces.test_suite_protocol import TestSuiteProtocol
from metadata.test_suite.runner.models import TestCaseResultResponse
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -17,6 +17,7 @@ import traceback
from typing import Optional
from metadata.config.common import ConfigModel
from metadata.data_quality.runner.models import TestCaseResultResponse
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection,
)
@ -24,7 +25,6 @@ from metadata.ingestion.api.common import Entity
from metadata.ingestion.api.sink import Sink
from metadata.ingestion.ometa.client import APIError
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.test_suite.runner.models import TestCaseResultResponse
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -26,7 +26,7 @@ from metadata.generated.schema.tests.basic import (
TestResultValue,
)
from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
from metadata.profiler.profiler.runner import QueryRunner
from metadata.profiler.processor.runner import QueryRunner
T = TypeVar("T", bound=Callable)
R = TypeVar("R")

View File

@ -19,13 +19,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -19,13 +19,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -19,13 +19,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -19,13 +19,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -19,13 +19,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -19,13 +19,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -20,13 +20,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -19,13 +19,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,11 +15,12 @@ Validator for column values to be between test case
import traceback
from abc import abstractmethod
from datetime import datetime
from datetime import date, datetime, time
from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
@ -27,7 +28,6 @@ from metadata.generated.schema.tests.basic import (
)
from metadata.profiler.metrics.registry import Metrics
from metadata.profiler.orm.registry import is_date_time
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn
from metadata.utils.time_utils import convert_timestamp
@ -41,6 +41,20 @@ MAX = "max"
class BaseColumnValuesToBeBetweenValidator(BaseTestValidator):
"""Validator for column values to be between test case"""
def _convert_date_to_datetime(
self, date_object: date, time_converter: time
) -> datetime:
"""Convert date object to datetime object
Args:
date_object (date): date object
time_converter (time): time converter to use one of time.min or time.max
Returns:
datetime:
"""
return datetime.combine(date_object, time_converter)
def run_validation(self) -> TestCaseResult:
"""Run validation for the given test case
@ -65,6 +79,11 @@ class BaseColumnValuesToBeBetweenValidator(BaseTestValidator):
],
)
if type(min_res) is date: # pylint: disable=unidiomatic-typecheck
min_res = self._convert_date_to_datetime(min_res, time.min)
if type(max_res) is date: # pylint: disable=unidiomatic-typecheck
max_res = self._convert_date_to_datetime(max_res, time.max)
min_bound = self.get_test_case_param_value(
self.test_case.parameterValues, # type: ignore
"minValue",

View File

@ -20,13 +20,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -20,13 +20,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.entity_link import get_table_fqn
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -19,13 +19,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -19,13 +19,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -19,13 +19,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -19,13 +19,13 @@ from typing import Union
from sqlalchemy import Column
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -16,13 +16,13 @@ Validator for column value length to be between test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValueLengthsToBeBetween import (
from metadata.data_quality.validations.column.base.columnValueLengthsToBeBetween import (
BaseColumnValueLengthsToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column value max to be between test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValueMaxToBeBetween import (
from metadata.data_quality.validations.column.base.columnValueMaxToBeBetween import (
BaseColumnValueMaxToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -16,13 +16,13 @@ Validator for column value mean to be between test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValueMeanToBeBetween import (
from metadata.data_quality.validations.column.base.columnValueMeanToBeBetween import (
BaseColumnValueMeanToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column value median to be between test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValueMedianToBeBetween import (
from metadata.data_quality.validations.column.base.columnValueMedianToBeBetween import (
BaseColumnValueMedianToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column value min to be between test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValueMinToBeBetween import (
from metadata.data_quality.validations.column.base.columnValueMinToBeBetween import (
BaseColumnValueMinToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column value stddev to be between test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValueStdDevToBeBetween import (
from metadata.data_quality.validations.column.base.columnValueStdDevToBeBetween import (
BaseColumnValueStdDevToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column value missing count to be equal test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesMissingCount import (
from metadata.data_quality.validations.column.base.columnValuesMissingCount import (
BaseColumnValuesMissingCountValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column values sum to be between test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesSumToBeBetween import (
from metadata.data_quality.validations.column.base.columnValuesSumToBeBetween import (
BaseColumnValuesSumToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column values to be between test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToBeBetween import (
from metadata.data_quality.validations.column.base.columnValuesToBeBetween import (
BaseColumnValuesToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column value to be in set test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToBeInSet import (
from metadata.data_quality.validations.column.base.columnValuesToBeInSet import (
BaseColumnValuesToBeInSetValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.logger import test_suite_logger
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column value to be not in set test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToBeNotInSet import (
from metadata.data_quality.validations.column.base.columnValuesToBeNotInSet import (
BaseColumnValuesToBeNotInSetValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column values to be not null test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToBeNotNull import (
from metadata.data_quality.validations.column.base.columnValuesToBeNotNull import (
BaseColumnValuesToBeNotNullValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column values to be unique test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToBeUnique import (
from metadata.data_quality.validations.column.base.columnValuesToBeUnique import (
BaseColumnValuesToBeUniqueValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column values to match regex test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToMatchRegex import (
from metadata.data_quality.validations.column.base.columnValuesToMatchRegex import (
BaseColumnValuesToMatchRegexValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -15,13 +15,13 @@ Validator for column values to not match regex test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToNotMatchRegex import (
from metadata.data_quality.validations.column.base.columnValuesToNotMatchRegex import (
BaseColumnValuesToNotMatchRegexValidator,
)
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.sqa_like_column import SQALikeColumn

View File

@ -18,11 +18,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValueLengthsToBeBetween import (
from metadata.data_quality.validations.column.base.columnValueLengthsToBeBetween import (
BaseColumnValueLengthsToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
class ColumnValueLengthsToBeBetweenValidator(

View File

@ -16,11 +16,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValueMaxToBeBetween import (
from metadata.data_quality.validations.column.base.columnValueMaxToBeBetween import (
BaseColumnValueMaxToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
class ColumnValueMaxToBeBetweenValidator(

View File

@ -17,11 +17,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValueMeanToBeBetween import (
from metadata.data_quality.validations.column.base.columnValueMeanToBeBetween import (
BaseColumnValueMeanToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
class ColumnValueMeanToBeBetweenValidator(

View File

@ -17,11 +17,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValueMedianToBeBetween import (
from metadata.data_quality.validations.column.base.columnValueMedianToBeBetween import (
BaseColumnValueMedianToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
class ColumnValueMedianToBeBetweenValidator(

View File

@ -17,11 +17,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValueMinToBeBetween import (
from metadata.data_quality.validations.column.base.columnValueMinToBeBetween import (
BaseColumnValueMinToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
class ColumnValueMinToBeBetweenValidator(

View File

@ -17,11 +17,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValueStdDevToBeBetween import (
from metadata.data_quality.validations.column.base.columnValueStdDevToBeBetween import (
BaseColumnValueStdDevToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
class ColumnValueStdDevToBeBetweenValidator(

View File

@ -17,11 +17,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesMissingCount import (
from metadata.data_quality.validations.column.base.columnValuesMissingCount import (
BaseColumnValuesMissingCountValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -17,11 +17,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesSumToBeBetween import (
from metadata.data_quality.validations.column.base.columnValuesSumToBeBetween import (
BaseColumnValuesSumToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
class ColumnValuesSumToBeBetweenValidator(

View File

@ -17,11 +17,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToBeBetween import (
from metadata.data_quality.validations.column.base.columnValuesToBeBetween import (
BaseColumnValuesToBeBetweenValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
class ColumnValuesToBeBetweenValidator(

View File

@ -17,11 +17,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToBeInSet import (
from metadata.data_quality.validations.column.base.columnValuesToBeInSet import (
BaseColumnValuesToBeInSetValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
class ColumnValuesToBeInSetValidator(

View File

@ -17,11 +17,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToBeNotInSet import (
from metadata.data_quality.validations.column.base.columnValuesToBeNotInSet import (
BaseColumnValuesToBeNotInSetValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
class ColumnValuesToBeNotInSetValidator(

View File

@ -17,11 +17,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToBeNotNull import (
from metadata.data_quality.validations.column.base.columnValuesToBeNotNull import (
BaseColumnValuesToBeNotNullValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -18,11 +18,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from sqlalchemy.orm.util import AliasedClass
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToBeUnique import (
from metadata.data_quality.validations.column.base.columnValuesToBeUnique import (
BaseColumnValuesToBeUniqueValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
class ColumnValuesToBeUniqueValidator(

View File

@ -18,11 +18,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from sqlalchemy.exc import CompileError
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToMatchRegex import (
from metadata.data_quality.validations.column.base.columnValuesToMatchRegex import (
BaseColumnValuesToMatchRegexValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -18,11 +18,13 @@ from typing import Optional
from sqlalchemy import Column, inspect
from sqlalchemy.exc import CompileError
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.column.base.columnValuesToNotMatchRegex import (
from metadata.data_quality.validations.column.base.columnValuesToNotMatchRegex import (
BaseColumnValuesToNotMatchRegexValidator,
)
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -20,7 +20,7 @@ from sqlalchemy.exc import SQLAlchemyError
from metadata.profiler.metrics.core import add_props
from metadata.profiler.metrics.registry import Metrics
from metadata.profiler.profiler.runner import QueryRunner
from metadata.profiler.processor.runner import QueryRunner
from metadata.utils.entity_link import get_decoded_column
from metadata.utils.logger import test_suite_logger

View File

@ -16,12 +16,12 @@ Validator for table column count to be between test case
import traceback
from abc import abstractmethod
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -16,12 +16,12 @@ Validator for table column count to be equal test case
import traceback
from abc import abstractmethod
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -16,12 +16,12 @@ Validator for table column nanme to exist test case
import traceback
from abc import abstractmethod
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -17,12 +17,12 @@ import collections
import traceback
from abc import abstractmethod
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -17,12 +17,12 @@ import traceback
from abc import abstractmethod
from typing import cast
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -16,13 +16,13 @@ Validator for table row count to be between test case
import traceback
from abc import abstractmethod
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -16,13 +16,13 @@ Validator for table row count to equal test case
import traceback
from abc import abstractmethod
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -17,12 +17,12 @@ import traceback
from abc import abstractmethod
from typing import cast
from metadata.data_quality.validations.base_test_handler import BaseTestValidator
from metadata.generated.schema.tests.basic import (
TestCaseResult,
TestCaseStatus,
TestResultValue,
)
from metadata.test_suite.validations.base_test_handler import BaseTestValidator
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -15,10 +15,10 @@ Validator for table column count to be between test case
from typing import Optional
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.test_suite.validations.table.base.tableColumnCountToBeBetween import (
from metadata.data_quality.validations.table.base.tableColumnCountToBeBetween import (
BaseTableColumnCountToBeBetweenValidator,
)

View File

@ -15,10 +15,10 @@ Validator for table column count to be equal test case
from typing import Optional
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.test_suite.validations.table.base.tableColumnCountToEqual import (
from metadata.data_quality.validations.table.base.tableColumnCountToEqual import (
BaseTableColumnCountToEqualValidator,
)
from metadata.utils.logger import test_suite_logger

View File

@ -13,10 +13,10 @@
Validator for table column name to exist test case
"""
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.test_suite.validations.table.base.tableColumnNameToExist import (
from metadata.data_quality.validations.table.base.tableColumnNameToExist import (
BaseTableColumnNameToExistValidator,
)
from metadata.utils.logger import test_suite_logger

View File

@ -14,10 +14,10 @@ Validator for table column name to match set test case
"""
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.test_suite.validations.table.base.tableColumnToMatchSet import (
from metadata.data_quality.validations.table.base.tableColumnToMatchSet import (
BaseTableColumnToMatchSetValidator,
)
from metadata.utils.logger import test_suite_logger

View File

@ -13,10 +13,10 @@
Validator for table custom SQL Query test case
"""
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.test_suite.validations.table.base.tableCustomSQLQuery import (
from metadata.data_quality.validations.table.base.tableCustomSQLQuery import (
BaseTableCustomSQLQueryValidator,
)

View File

@ -15,13 +15,13 @@ Validator for table row count to be between test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.test_suite.validations.table.base.tableRowCountToBeBetween import (
from metadata.data_quality.validations.table.base.tableRowCountToBeBetween import (
BaseTableRowCountToBeBetweenValidator,
)
from metadata.profiler.metrics.registry import Metrics
class TableRowCountToBeBetweenValidator(

View File

@ -15,13 +15,13 @@ Validator for table row count to be equal test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.test_suite.validations.table.base.tableRowCountToEqual import (
from metadata.data_quality.validations.table.base.tableRowCountToEqual import (
BaseTableRowCountToEqualValidator,
)
from metadata.profiler.metrics.registry import Metrics
class TableRowCountToEqualValidator(

View File

@ -17,10 +17,10 @@ from datetime import datetime
from dateutil.relativedelta import relativedelta
from metadata.test_suite.validations.mixins.pandas_validator_mixin import (
from metadata.data_quality.validations.mixins.pandas_validator_mixin import (
PandasValidatorMixin,
)
from metadata.test_suite.validations.table.base.tableRowInsertedCountToBeBetween import (
from metadata.data_quality.validations.table.base.tableRowInsertedCountToBeBetween import (
BaseTableRowInsertedCountToBeBetweenValidator,
)

View File

@ -17,8 +17,10 @@ from typing import Optional
from sqlalchemy import inspect
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.test_suite.validations.table.base.tableColumnCountToBeBetween import (
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.data_quality.validations.table.base.tableColumnCountToBeBetween import (
BaseTableColumnCountToBeBetweenValidator,
)

View File

@ -17,8 +17,10 @@ from typing import Optional
from sqlalchemy import inspect
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.test_suite.validations.table.base.tableColumnCountToEqual import (
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.data_quality.validations.table.base.tableColumnCountToEqual import (
BaseTableColumnCountToEqualValidator,
)

View File

@ -15,8 +15,10 @@ Validator for table column nanme to exist test case
from sqlalchemy import inspect
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.test_suite.validations.table.base.tableColumnNameToExist import (
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.data_quality.validations.table.base.tableColumnNameToExist import (
BaseTableColumnNameToExistValidator,
)
from metadata.utils.logger import test_suite_logger

View File

@ -18,8 +18,10 @@ from typing import Optional
from sqlalchemy import inspect
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.test_suite.validations.table.base.tableColumnToMatchSet import (
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.data_quality.validations.table.base.tableColumnToMatchSet import (
BaseTableColumnToMatchSetValidator,
)
from metadata.utils.logger import test_suite_logger

View File

@ -15,8 +15,10 @@ Validator for table custom SQL Query test case
from sqlalchemy import text
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.test_suite.validations.table.base.tableCustomSQLQuery import (
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.data_quality.validations.table.base.tableCustomSQLQuery import (
BaseTableCustomSQLQueryValidator,
)

View File

@ -15,11 +15,13 @@ Validator for table row count to be between test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.test_suite.validations.table.base.tableRowCountToBeBetween import (
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.data_quality.validations.table.base.tableRowCountToBeBetween import (
BaseTableRowCountToBeBetweenValidator,
)
from metadata.profiler.metrics.registry import Metrics
from metadata.utils.logger import test_suite_logger
logger = test_suite_logger()

View File

@ -15,11 +15,13 @@ Validator for table row inserted count to be between test case
from typing import Optional
from metadata.profiler.metrics.registry import Metrics
from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin
from metadata.test_suite.validations.table.base.tableRowCountToEqual import (
from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
SQAValidatorMixin,
)
from metadata.data_quality.validations.table.base.tableRowCountToEqual import (
BaseTableRowCountToEqualValidator,
)
from metadata.profiler.metrics.registry import Metrics
class TableRowCountToEqualValidator(

Some files were not shown because too many files have changed in this diff Show More