diff --git a/ingestion/operators/docker/main.py b/ingestion/operators/docker/main.py index 1132785d3f0..71595b8855f 100644 --- a/ingestion/operators/docker/main.py +++ b/ingestion/operators/docker/main.py @@ -16,12 +16,12 @@ import os import yaml from metadata.data_insight.api.workflow import DataInsightWorkflow +from metadata.data_quality.api.workflow import TestSuiteWorkflow from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import ( PipelineType, ) from metadata.ingestion.api.workflow import Workflow from metadata.profiler.api.workflow import ProfilerWorkflow -from metadata.test_suite.api.workflow import TestSuiteWorkflow WORKFLOW_MAP = { PipelineType.metadata.value: Workflow, diff --git a/ingestion/src/metadata/cli/dataquality.py b/ingestion/src/metadata/cli/dataquality.py index ed98e0eda71..a7cb24e41d6 100644 --- a/ingestion/src/metadata/cli/dataquality.py +++ b/ingestion/src/metadata/cli/dataquality.py @@ -17,7 +17,7 @@ import sys import traceback from metadata.config.common import load_config_file -from metadata.test_suite.api.workflow import TestSuiteWorkflow +from metadata.data_quality.api.workflow import TestSuiteWorkflow from metadata.utils.logger import cli_logger from metadata.utils.workflow_output_handler import WorkflowType, print_init_error diff --git a/ingestion/src/metadata/test_suite/README.md b/ingestion/src/metadata/data_quality/README.md similarity index 100% rename from ingestion/src/metadata/test_suite/README.md rename to ingestion/src/metadata/data_quality/README.md diff --git a/ingestion/src/metadata/test_suite/__init__.py b/ingestion/src/metadata/data_quality/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/__init__.py rename to ingestion/src/metadata/data_quality/__init__.py diff --git a/ingestion/src/metadata/interfaces/__init__.py b/ingestion/src/metadata/data_quality/api/__init__.py similarity index 100% rename from ingestion/src/metadata/interfaces/__init__.py rename to ingestion/src/metadata/data_quality/api/__init__.py diff --git a/ingestion/src/metadata/test_suite/api/models.py b/ingestion/src/metadata/data_quality/api/models.py similarity index 100% rename from ingestion/src/metadata/test_suite/api/models.py rename to ingestion/src/metadata/data_quality/api/models.py diff --git a/ingestion/src/metadata/test_suite/api/workflow.py b/ingestion/src/metadata/data_quality/api/workflow.py similarity index 98% rename from ingestion/src/metadata/test_suite/api/workflow.py rename to ingestion/src/metadata/data_quality/api/workflow.py index 526a6610f67..cc430468970 100644 --- a/ingestion/src/metadata/test_suite/api/workflow.py +++ b/ingestion/src/metadata/data_quality/api/workflow.py @@ -25,6 +25,18 @@ from pydantic import BaseModel, ValidationError from sqlalchemy import MetaData from metadata.config.common import WorkflowExecutionError +from metadata.data_quality.api.models import ( + TestCaseDefinition, + TestSuiteDefinition, + TestSuiteProcessorConfig, +) +from metadata.data_quality.interface.pandas.pandas_test_suite_interface import ( + PandasTestSuiteInterface, +) +from metadata.data_quality.interface.sqlalchemy.sqa_test_suite_interface import ( + SQATestSuiteInterface, +) +from metadata.data_quality.runner.core import DataTestsRunner from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest from metadata.generated.schema.entity.data.table import PartitionProfilerConfig, Table @@ -53,17 +65,7 @@ from metadata.ingestion.ometa.client_utils import create_ometa_client from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.connections import get_connection from metadata.ingestion.source.database.datalake.metadata import ometa_to_dataframe -from metadata.interfaces.datalake.datalake_test_suite_interface import ( - DataLakeTestSuiteInterface, -) -from metadata.interfaces.sqalchemy.sqa_test_suite_interface import SQATestSuiteInterface from metadata.profiler.api.models import ProfileSampleConfig -from metadata.test_suite.api.models import ( - TestCaseDefinition, - TestSuiteDefinition, - TestSuiteProcessorConfig, -) -from metadata.test_suite.runner.core import DataTestsRunner from metadata.utils import entity_link from metadata.utils.importer import get_sink from metadata.utils.logger import test_suite_logger @@ -127,7 +129,7 @@ class TestSuiteWorkflow(WorkflowStatusMixin): sink_type=self.config.sink.type, sink_config=self.config.sink, metadata_config=self.metadata_config, - from_="test_suite", + from_="data_quality", ) @classmethod @@ -283,7 +285,7 @@ class TestSuiteWorkflow(WorkflowStatusMixin): table_sample_query=table_sample_query, table_partition_config=table_partition_config, ) - return DataLakeTestSuiteInterface( + return PandasTestSuiteInterface( service_connection_config=service_connection_config, ometa_client=self.client, df=ometa_to_dataframe( diff --git a/ingestion/src/metadata/interfaces/datalake/__init__.py b/ingestion/src/metadata/data_quality/interface/__init__.py similarity index 100% rename from ingestion/src/metadata/interfaces/datalake/__init__.py rename to ingestion/src/metadata/data_quality/interface/__init__.py diff --git a/ingestion/src/metadata/interfaces/datalake/mixins/__init__.py b/ingestion/src/metadata/data_quality/interface/pandas/__init__.py similarity index 100% rename from ingestion/src/metadata/interfaces/datalake/mixins/__init__.py rename to ingestion/src/metadata/data_quality/interface/pandas/__init__.py diff --git a/ingestion/src/metadata/interfaces/datalake/datalake_test_suite_interface.py b/ingestion/src/metadata/data_quality/interface/pandas/pandas_test_suite_interface.py similarity index 90% rename from ingestion/src/metadata/interfaces/datalake/datalake_test_suite_interface.py rename to ingestion/src/metadata/data_quality/interface/pandas/pandas_test_suite_interface.py index 396d61c00ff..8bb43f537bf 100644 --- a/ingestion/src/metadata/interfaces/datalake/datalake_test_suite_interface.py +++ b/ingestion/src/metadata/data_quality/interface/pandas/pandas_test_suite_interface.py @@ -16,6 +16,8 @@ supporting sqlalchemy abstraction layer from datetime import datetime, timezone from typing import Optional +from metadata.data_quality.interface.test_suite_protocol import TestSuiteProtocol +from metadata.data_quality.validations.validator import Validator from metadata.generated.schema.entity.services.connections.database.datalakeConnection import ( DatalakeConnection, ) @@ -23,16 +25,14 @@ from metadata.generated.schema.tests.basic import TestCaseResult from metadata.generated.schema.tests.testCase import TestCase from metadata.generated.schema.tests.testDefinition import TestDefinition from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.interfaces.datalake.mixins.pandas_mixin import PandasInterfaceMixin -from metadata.interfaces.test_suite_protocol import TestSuiteProtocol -from metadata.test_suite.validations.validator import Validator +from metadata.mixins.pandas.pandas_mixin import PandasInterfaceMixin from metadata.utils.importer import import_test_case_class from metadata.utils.logger import test_suite_logger logger = test_suite_logger() -class DataLakeTestSuiteInterface(TestSuiteProtocol, PandasInterfaceMixin): +class PandasTestSuiteInterface(TestSuiteProtocol, PandasInterfaceMixin): """ Sequential interface protocol for testSuite and Profiler. This class implements specific operations needed to run profiler and test suite workflow diff --git a/ingestion/src/metadata/interfaces/sqalchemy/__init__.py b/ingestion/src/metadata/data_quality/interface/sqlalchemy/__init__.py similarity index 100% rename from ingestion/src/metadata/interfaces/sqalchemy/__init__.py rename to ingestion/src/metadata/data_quality/interface/sqlalchemy/__init__.py diff --git a/ingestion/src/metadata/interfaces/sqalchemy/sqa_test_suite_interface.py b/ingestion/src/metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py similarity index 94% rename from ingestion/src/metadata/interfaces/sqalchemy/sqa_test_suite_interface.py rename to ingestion/src/metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py index f6e8403bded..5e665b4576b 100644 --- a/ingestion/src/metadata/interfaces/sqalchemy/sqa_test_suite_interface.py +++ b/ingestion/src/metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py @@ -21,6 +21,8 @@ from sqlalchemy import MetaData from sqlalchemy.orm import DeclarativeMeta from sqlalchemy.orm.util import AliasedClass +from metadata.data_quality.interface.test_suite_protocol import TestSuiteProtocol +from metadata.data_quality.validations.validator import Validator from metadata.generated.schema.entity.data.table import PartitionProfilerConfig, Table from metadata.generated.schema.entity.services.databaseService import DatabaseConnection from metadata.generated.schema.tests.basic import TestCaseResult @@ -29,12 +31,10 @@ from metadata.generated.schema.tests.testDefinition import TestDefinition from metadata.ingestion.connections.session import create_and_bind_session from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.connections import get_connection -from metadata.interfaces.sqalchemy.mixins.sqa_mixin import SQAInterfaceMixin -from metadata.interfaces.test_suite_protocol import TestSuiteProtocol +from metadata.mixins.sqalchemy.sqa_mixin import SQAInterfaceMixin from metadata.profiler.api.models import ProfileSampleConfig -from metadata.profiler.profiler.runner import QueryRunner -from metadata.profiler.profiler.sampler import Sampler -from metadata.test_suite.validations.validator import Validator +from metadata.profiler.processor.runner import QueryRunner +from metadata.profiler.processor.sampler import Sampler from metadata.utils.constants import TEN_MIN from metadata.utils.importer import import_test_case_class from metadata.utils.logger import test_suite_logger diff --git a/ingestion/src/metadata/interfaces/test_suite_protocol.py b/ingestion/src/metadata/data_quality/interface/test_suite_protocol.py similarity index 100% rename from ingestion/src/metadata/interfaces/test_suite_protocol.py rename to ingestion/src/metadata/data_quality/interface/test_suite_protocol.py diff --git a/ingestion/src/metadata/interfaces/sqalchemy/mixins/__init__.py b/ingestion/src/metadata/data_quality/runner/__init__.py similarity index 100% rename from ingestion/src/metadata/interfaces/sqalchemy/mixins/__init__.py rename to ingestion/src/metadata/data_quality/runner/__init__.py diff --git a/ingestion/src/metadata/test_suite/runner/core.py b/ingestion/src/metadata/data_quality/runner/core.py similarity index 90% rename from ingestion/src/metadata/test_suite/runner/core.py rename to ingestion/src/metadata/data_quality/runner/core.py index 23196568cef..4572c86c316 100644 --- a/ingestion/src/metadata/test_suite/runner/core.py +++ b/ingestion/src/metadata/data_quality/runner/core.py @@ -14,9 +14,9 @@ Main class to run data tests """ +from metadata.data_quality.interface.test_suite_protocol import TestSuiteProtocol +from metadata.data_quality.runner.models import TestCaseResultResponse from metadata.generated.schema.tests.testCase import TestCase -from metadata.interfaces.test_suite_protocol import TestSuiteProtocol -from metadata.test_suite.runner.models import TestCaseResultResponse from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/runner/models.py b/ingestion/src/metadata/data_quality/runner/models.py similarity index 100% rename from ingestion/src/metadata/test_suite/runner/models.py rename to ingestion/src/metadata/data_quality/runner/models.py diff --git a/ingestion/src/metadata/profiler/profiler/__init__.py b/ingestion/src/metadata/data_quality/sink/__init__.py similarity index 100% rename from ingestion/src/metadata/profiler/profiler/__init__.py rename to ingestion/src/metadata/data_quality/sink/__init__.py diff --git a/ingestion/src/metadata/test_suite/sink/metadata_rest.py b/ingestion/src/metadata/data_quality/sink/metadata_rest.py similarity index 97% rename from ingestion/src/metadata/test_suite/sink/metadata_rest.py rename to ingestion/src/metadata/data_quality/sink/metadata_rest.py index 49c2dbfef98..423d11c3139 100644 --- a/ingestion/src/metadata/test_suite/sink/metadata_rest.py +++ b/ingestion/src/metadata/data_quality/sink/metadata_rest.py @@ -17,6 +17,7 @@ import traceback from typing import Optional from metadata.config.common import ConfigModel +from metadata.data_quality.runner.models import TestCaseResultResponse from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) @@ -24,7 +25,6 @@ from metadata.ingestion.api.common import Entity from metadata.ingestion.api.sink import Sink from metadata.ingestion.ometa.client import APIError from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.test_suite.runner.models import TestCaseResultResponse from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/README.md b/ingestion/src/metadata/data_quality/validations/README.md similarity index 100% rename from ingestion/src/metadata/test_suite/validations/README.md rename to ingestion/src/metadata/data_quality/validations/README.md diff --git a/ingestion/src/metadata/test_suite/validations/__init__.py b/ingestion/src/metadata/data_quality/validations/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/validations/__init__.py rename to ingestion/src/metadata/data_quality/validations/__init__.py diff --git a/ingestion/src/metadata/test_suite/validations/base_test_handler.py b/ingestion/src/metadata/data_quality/validations/base_test_handler.py similarity index 98% rename from ingestion/src/metadata/test_suite/validations/base_test_handler.py rename to ingestion/src/metadata/data_quality/validations/base_test_handler.py index e8e092c5974..3e423f55b6a 100644 --- a/ingestion/src/metadata/test_suite/validations/base_test_handler.py +++ b/ingestion/src/metadata/data_quality/validations/base_test_handler.py @@ -26,7 +26,7 @@ from metadata.generated.schema.tests.basic import ( TestResultValue, ) from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue -from metadata.profiler.profiler.runner import QueryRunner +from metadata.profiler.processor.runner import QueryRunner T = TypeVar("T", bound=Callable) R = TypeVar("R") diff --git a/ingestion/src/metadata/test_suite/validations/column/__init__.py b/ingestion/src/metadata/data_quality/validations/column/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/validations/column/__init__.py rename to ingestion/src/metadata/data_quality/validations/column/__init__.py diff --git a/ingestion/src/metadata/test_suite/validations/column/base/__init__.py b/ingestion/src/metadata/data_quality/validations/column/base/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/validations/column/base/__init__.py rename to ingestion/src/metadata/data_quality/validations/column/base/__init__.py diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValueLengthsToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py similarity index 97% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValueLengthsToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py index 623ca7a32c7..e946f8d4db1 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValueLengthsToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py @@ -19,13 +19,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValueMaxToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValueMaxToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py index df2e395c733..731546af704 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValueMaxToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py @@ -19,13 +19,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValueMeanToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValueMeanToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py index d78b3dc6fe7..6216a0ad089 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValueMeanToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py @@ -19,13 +19,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValueMedianToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValueMedianToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py index 25875ccfe82..a0089afc75a 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValueMedianToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py @@ -19,13 +19,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValueMinToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValueMinToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py index a8e9b0db2a1..4f90f685276 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValueMinToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py @@ -19,13 +19,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValueStdDevToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValueStdDevToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py index 3fe2f1db31b..b1da15f537f 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValueStdDevToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py @@ -19,13 +19,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesMissingCount.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesMissingCount.py similarity index 97% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValuesMissingCount.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValuesMissingCount.py index 75497cc3e6c..f3d61223460 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesMissingCount.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesMissingCount.py @@ -20,13 +20,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesSumToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValuesSumToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py index dab27f51e60..10ea974d07d 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesSumToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py @@ -19,13 +19,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeBetween.py similarity index 81% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeBetween.py index d256f96a9e1..f4dd54bf38b 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeBetween.py @@ -15,11 +15,12 @@ Validator for column values to be between test case import traceback from abc import abstractmethod -from datetime import datetime +from datetime import date, datetime, time from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, @@ -27,7 +28,6 @@ from metadata.generated.schema.tests.basic import ( ) from metadata.profiler.metrics.registry import Metrics from metadata.profiler.orm.registry import is_date_time -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn from metadata.utils.time_utils import convert_timestamp @@ -41,6 +41,20 @@ MAX = "max" class BaseColumnValuesToBeBetweenValidator(BaseTestValidator): """Validator for column values to be between test case""" + def _convert_date_to_datetime( + self, date_object: date, time_converter: time + ) -> datetime: + """Convert date object to datetime object + + Args: + date_object (date): date object + time_converter (time): time converter to use one of time.min or time.max + + Returns: + datetime: + """ + return datetime.combine(date_object, time_converter) + def run_validation(self) -> TestCaseResult: """Run validation for the given test case @@ -65,6 +79,11 @@ class BaseColumnValuesToBeBetweenValidator(BaseTestValidator): ], ) + if type(min_res) is date: # pylint: disable=unidiomatic-typecheck + min_res = self._convert_date_to_datetime(min_res, time.min) + if type(max_res) is date: # pylint: disable=unidiomatic-typecheck + max_res = self._convert_date_to_datetime(max_res, time.max) + min_bound = self.get_test_case_param_value( self.test_case.parameterValues, # type: ignore "minValue", diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeInSet.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeInSet.py similarity index 97% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeInSet.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeInSet.py index 23af442b713..ed6a5b1a00d 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeInSet.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeInSet.py @@ -20,13 +20,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeNotInSet.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py similarity index 97% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeNotInSet.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py index 40f9661061d..d1c07f6fdde 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeNotInSet.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py @@ -20,13 +20,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.entity_link import get_table_fqn from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeNotNull.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeNotNull.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py index 679bf7aa701..b9cf74dfdb4 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeNotNull.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py @@ -19,13 +19,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeUnique.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeUnique.py similarity index 97% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeUnique.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeUnique.py index f4988cc5f1b..66bf1148309 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToBeUnique.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeUnique.py @@ -19,13 +19,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToMatchRegex.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py similarity index 97% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValuesToMatchRegex.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py index 238e1e43917..dfcfd1a0983 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToMatchRegex.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py @@ -19,13 +19,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToNotMatchRegex.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py similarity index 97% rename from ingestion/src/metadata/test_suite/validations/column/base/columnValuesToNotMatchRegex.py rename to ingestion/src/metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py index 81ecf1122dc..afc13ff6436 100644 --- a/ingestion/src/metadata/test_suite/validations/column/base/columnValuesToNotMatchRegex.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py @@ -19,13 +19,13 @@ from typing import Union from sqlalchemy import Column +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/__init__.py b/ingestion/src/metadata/data_quality/validations/column/pandas/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/validations/column/pandas/__init__.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/__init__.py diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValueLengthsToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValueLengthsToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py index 74137a12c02..82e62460611 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValueLengthsToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py @@ -16,13 +16,13 @@ Validator for column value length to be between test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValueLengthsToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValueLengthsToBeBetween import ( BaseColumnValueLengthsToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValueMaxToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValueMaxToBeBetween.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValueMaxToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValueMaxToBeBetween.py index 0fc4341a088..17878b40443 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValueMaxToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValueMaxToBeBetween.py @@ -15,13 +15,13 @@ Validator for column value max to be between test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValueMaxToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValueMaxToBeBetween import ( BaseColumnValueMaxToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValueMeanToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValueMeanToBeBetween.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValueMeanToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValueMeanToBeBetween.py index 26b773d7207..8505810d305 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValueMeanToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValueMeanToBeBetween.py @@ -16,13 +16,13 @@ Validator for column value mean to be between test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValueMeanToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValueMeanToBeBetween import ( BaseColumnValueMeanToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValueMedianToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValueMedianToBeBetween.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValueMedianToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValueMedianToBeBetween.py index 6df1019e397..530aad4bea7 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValueMedianToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValueMedianToBeBetween.py @@ -15,13 +15,13 @@ Validator for column value median to be between test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValueMedianToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValueMedianToBeBetween import ( BaseColumnValueMedianToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValueMinToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValueMinToBeBetween.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValueMinToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValueMinToBeBetween.py index ecbfb8bb49d..b96dac47b58 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValueMinToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValueMinToBeBetween.py @@ -15,13 +15,13 @@ Validator for column value min to be between test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValueMinToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValueMinToBeBetween import ( BaseColumnValueMinToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValueStdDevToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValueStdDevToBeBetween.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValueStdDevToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValueStdDevToBeBetween.py index 9e18e840f65..6a77297641b 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValueStdDevToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValueStdDevToBeBetween.py @@ -15,13 +15,13 @@ Validator for column value stddev to be between test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValueStdDevToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValueStdDevToBeBetween import ( BaseColumnValueStdDevToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesMissingCount.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesMissingCount.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesMissingCount.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesMissingCount.py index d75322bac05..99e5d92d990 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesMissingCount.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesMissingCount.py @@ -15,13 +15,13 @@ Validator for column value missing count to be equal test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesMissingCount import ( +from metadata.data_quality.validations.column.base.columnValuesMissingCount import ( BaseColumnValuesMissingCountValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesSumToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesSumToBeBetween.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesSumToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesSumToBeBetween.py index 26c3f8ab355..f9b15bec1ad 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesSumToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesSumToBeBetween.py @@ -15,13 +15,13 @@ Validator for column values sum to be between test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesSumToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValuesSumToBeBetween import ( BaseColumnValuesSumToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py index df79cd489f0..fa3c9c5f5c6 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py @@ -15,13 +15,13 @@ Validator for column values to be between test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValuesToBeBetween import ( BaseColumnValuesToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeInSet.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py similarity index 91% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeInSet.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py index 2f6336f5277..06376182b5c 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeInSet.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py @@ -15,13 +15,13 @@ Validator for column value to be in set test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToBeInSet import ( +from metadata.data_quality.validations.column.base.columnValuesToBeInSet import ( BaseColumnValuesToBeInSetValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.logger import test_suite_logger from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeNotInSet.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeNotInSet.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py index 5e680ef5074..c9c74f65949 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeNotInSet.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py @@ -15,13 +15,13 @@ Validator for column value to be not in set test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToBeNotInSet import ( +from metadata.data_quality.validations.column.base.columnValuesToBeNotInSet import ( BaseColumnValuesToBeNotInSetValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeNotNull.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeNotNull.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py index 369d5b30fd0..226dbd1d8ac 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeNotNull.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py @@ -15,13 +15,13 @@ Validator for column values to be not null test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToBeNotNull import ( +from metadata.data_quality.validations.column.base.columnValuesToBeNotNull import ( BaseColumnValuesToBeNotNullValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeUnique.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeUnique.py similarity index 91% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeUnique.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeUnique.py index ab818be9ae7..238e419dffc 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToBeUnique.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToBeUnique.py @@ -15,13 +15,13 @@ Validator for column values to be unique test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToBeUnique import ( +from metadata.data_quality.validations.column.base.columnValuesToBeUnique import ( BaseColumnValuesToBeUniqueValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToMatchRegex.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToMatchRegex.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py index 199cb3fc6d0..c7fa2e1d5d0 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToMatchRegex.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py @@ -15,13 +15,13 @@ Validator for column values to match regex test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToMatchRegex import ( +from metadata.data_quality.validations.column.base.columnValuesToMatchRegex import ( BaseColumnValuesToMatchRegexValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToNotMatchRegex.py b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToNotMatchRegex.py rename to ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py index 5d3dfe0c666..ce95c852a91 100644 --- a/ingestion/src/metadata/test_suite/validations/column/pandas/columnValuesToNotMatchRegex.py +++ b/ingestion/src/metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py @@ -15,13 +15,13 @@ Validator for column values to not match regex test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToNotMatchRegex import ( +from metadata.data_quality.validations.column.base.columnValuesToNotMatchRegex import ( BaseColumnValuesToNotMatchRegexValidator, ) -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_like_column import SQALikeColumn diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/__init__.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/__init__.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/__init__.py diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py similarity index 89% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py index cd92638b347..788f9ac1fe4 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py @@ -18,11 +18,13 @@ from typing import Optional from sqlalchemy import Column, inspect -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValueLengthsToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValueLengthsToBeBetween import ( BaseColumnValueLengthsToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics class ColumnValueLengthsToBeBetweenValidator( diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueMaxToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py similarity index 89% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueMaxToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py index 16100bf54b6..fc0958a37db 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py @@ -16,11 +16,13 @@ from typing import Optional from sqlalchemy import Column, inspect -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValueMaxToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValueMaxToBeBetween import ( BaseColumnValueMaxToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics class ColumnValueMaxToBeBetweenValidator( diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueMeanToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py similarity index 89% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueMeanToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py index 75eb5e089c9..aa01d6b3a3f 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py @@ -17,11 +17,13 @@ from typing import Optional from sqlalchemy import Column, inspect -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValueMeanToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValueMeanToBeBetween import ( BaseColumnValueMeanToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics class ColumnValueMeanToBeBetweenValidator( diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueMedianToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py similarity index 89% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueMedianToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py index ae61d4e9d90..e1d81956ce3 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py @@ -17,11 +17,13 @@ from typing import Optional from sqlalchemy import Column, inspect -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValueMedianToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValueMedianToBeBetween import ( BaseColumnValueMedianToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics class ColumnValueMedianToBeBetweenValidator( diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueMinToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py similarity index 89% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueMinToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py index d0803d8fab1..f84b0c58a5f 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueMinToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py @@ -17,11 +17,13 @@ from typing import Optional from sqlalchemy import Column, inspect -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValueMinToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValueMinToBeBetween import ( BaseColumnValueMinToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics class ColumnValueMinToBeBetweenValidator( diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py similarity index 89% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py index 18bd5d65a36..cf4dc5da836 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py @@ -17,11 +17,13 @@ from typing import Optional from sqlalchemy import Column, inspect -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValueStdDevToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValueStdDevToBeBetween import ( BaseColumnValueStdDevToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics class ColumnValueStdDevToBeBetweenValidator( diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesMissingCount.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py similarity index 90% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesMissingCount.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py index ce5ac799dee..89e5e7c83c8 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesMissingCount.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py @@ -17,11 +17,13 @@ from typing import Optional from sqlalchemy import Column, inspect -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesMissingCount import ( +from metadata.data_quality.validations.column.base.columnValuesMissingCount import ( BaseColumnValuesMissingCountValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesSumToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py similarity index 89% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesSumToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py index 47c6faf03e3..617fbc10c3d 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py @@ -17,11 +17,13 @@ from typing import Optional from sqlalchemy import Column, inspect -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesSumToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValuesSumToBeBetween import ( BaseColumnValuesSumToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics class ColumnValuesSumToBeBetweenValidator( diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py similarity index 89% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py index fb18e42034b..3141c917cda 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py @@ -17,11 +17,13 @@ from typing import Optional from sqlalchemy import Column, inspect -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToBeBetween import ( +from metadata.data_quality.validations.column.base.columnValuesToBeBetween import ( BaseColumnValuesToBeBetweenValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics class ColumnValuesToBeBetweenValidator( diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeInSet.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py similarity index 89% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeInSet.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py index 9096eb3ed24..f1b5cf9b3cf 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeInSet.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py @@ -17,11 +17,13 @@ from typing import Optional from sqlalchemy import Column, inspect -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToBeInSet import ( +from metadata.data_quality.validations.column.base.columnValuesToBeInSet import ( BaseColumnValuesToBeInSetValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics class ColumnValuesToBeInSetValidator( diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeNotInSet.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py similarity index 89% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeNotInSet.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py index 9891ec25a56..c2486bca481 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py @@ -17,11 +17,13 @@ from typing import Optional from sqlalchemy import Column, inspect -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToBeNotInSet import ( +from metadata.data_quality.validations.column.base.columnValuesToBeNotInSet import ( BaseColumnValuesToBeNotInSetValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics class ColumnValuesToBeNotInSetValidator( diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeNotNull.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py similarity index 89% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeNotNull.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py index 5a43d332a7a..94b99e5f2c6 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeNotNull.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py @@ -17,11 +17,13 @@ from typing import Optional from sqlalchemy import Column, inspect -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToBeNotNull import ( +from metadata.data_quality.validations.column.base.columnValuesToBeNotNull import ( BaseColumnValuesToBeNotNullValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeUnique.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py similarity index 92% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeUnique.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py index c021d0c113c..583590928d9 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToBeUnique.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py @@ -18,11 +18,13 @@ from typing import Optional from sqlalchemy import Column, inspect from sqlalchemy.orm.util import AliasedClass -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToBeUnique import ( +from metadata.data_quality.validations.column.base.columnValuesToBeUnique import ( BaseColumnValuesToBeUniqueValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics class ColumnValuesToBeUniqueValidator( diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToMatchRegex.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py similarity index 91% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToMatchRegex.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py index abce6593708..3539aa6ed6e 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToMatchRegex.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py @@ -18,11 +18,13 @@ from typing import Optional from sqlalchemy import Column, inspect from sqlalchemy.exc import CompileError -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToMatchRegex import ( +from metadata.data_quality.validations.column.base.columnValuesToMatchRegex import ( BaseColumnValuesToMatchRegexValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py similarity index 91% rename from ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py rename to ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py index fa66b4db600..97179d7d002 100644 --- a/ingestion/src/metadata/test_suite/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +++ b/ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py @@ -18,11 +18,13 @@ from typing import Optional from sqlalchemy import Column, inspect from sqlalchemy.exc import CompileError -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.column.base.columnValuesToNotMatchRegex import ( +from metadata.data_quality.validations.column.base.columnValuesToNotMatchRegex import ( BaseColumnValuesToNotMatchRegexValidator, ) -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/api/__init__.py b/ingestion/src/metadata/data_quality/validations/mixins/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/api/__init__.py rename to ingestion/src/metadata/data_quality/validations/mixins/__init__.py diff --git a/ingestion/src/metadata/test_suite/validations/mixins/pandas_validator_mixin.py b/ingestion/src/metadata/data_quality/validations/mixins/pandas_validator_mixin.py similarity index 100% rename from ingestion/src/metadata/test_suite/validations/mixins/pandas_validator_mixin.py rename to ingestion/src/metadata/data_quality/validations/mixins/pandas_validator_mixin.py diff --git a/ingestion/src/metadata/test_suite/validations/mixins/sqa_validator_mixin.py b/ingestion/src/metadata/data_quality/validations/mixins/sqa_validator_mixin.py similarity index 97% rename from ingestion/src/metadata/test_suite/validations/mixins/sqa_validator_mixin.py rename to ingestion/src/metadata/data_quality/validations/mixins/sqa_validator_mixin.py index edb3e53aeeb..91a097cd5f0 100644 --- a/ingestion/src/metadata/test_suite/validations/mixins/sqa_validator_mixin.py +++ b/ingestion/src/metadata/data_quality/validations/mixins/sqa_validator_mixin.py @@ -20,7 +20,7 @@ from sqlalchemy.exc import SQLAlchemyError from metadata.profiler.metrics.core import add_props from metadata.profiler.metrics.registry import Metrics -from metadata.profiler.profiler.runner import QueryRunner +from metadata.profiler.processor.runner import QueryRunner from metadata.utils.entity_link import get_decoded_column from metadata.utils.logger import test_suite_logger diff --git a/ingestion/src/metadata/test_suite/validations/table/__init__.py b/ingestion/src/metadata/data_quality/validations/table/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/validations/table/__init__.py rename to ingestion/src/metadata/data_quality/validations/table/__init__.py diff --git a/ingestion/src/metadata/test_suite/validations/table/base/__init__.py b/ingestion/src/metadata/data_quality/validations/table/base/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/validations/table/base/__init__.py rename to ingestion/src/metadata/data_quality/validations/table/base/__init__.py diff --git a/ingestion/src/metadata/test_suite/validations/table/base/tableColumnCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToBeBetween.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/table/base/tableColumnCountToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToBeBetween.py index 1f94b5cb402..530d0ee4ca4 100644 --- a/ingestion/src/metadata/test_suite/validations/table/base/tableColumnCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToBeBetween.py @@ -16,12 +16,12 @@ Validator for table column count to be between test case import traceback from abc import abstractmethod +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/table/base/tableColumnCountToEqual.py b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToEqual.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/table/base/tableColumnCountToEqual.py rename to ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToEqual.py index 54fc54f7fa3..5a30291f899 100644 --- a/ingestion/src/metadata/test_suite/validations/table/base/tableColumnCountToEqual.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToEqual.py @@ -16,12 +16,12 @@ Validator for table column count to be equal test case import traceback from abc import abstractmethod +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/table/base/tableColumnNameToExist.py b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnNameToExist.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/table/base/tableColumnNameToExist.py rename to ingestion/src/metadata/data_quality/validations/table/base/tableColumnNameToExist.py index ef82b8a3cf8..aad93dcbc69 100644 --- a/ingestion/src/metadata/test_suite/validations/table/base/tableColumnNameToExist.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnNameToExist.py @@ -16,12 +16,12 @@ Validator for table column nanme to exist test case import traceback from abc import abstractmethod +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/table/base/tableColumnToMatchSet.py b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnToMatchSet.py similarity index 97% rename from ingestion/src/metadata/test_suite/validations/table/base/tableColumnToMatchSet.py rename to ingestion/src/metadata/data_quality/validations/table/base/tableColumnToMatchSet.py index 981472a7017..d135109351a 100644 --- a/ingestion/src/metadata/test_suite/validations/table/base/tableColumnToMatchSet.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnToMatchSet.py @@ -17,12 +17,12 @@ import collections import traceback from abc import abstractmethod +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/table/base/tableCustomSQLQuery.py b/ingestion/src/metadata/data_quality/validations/table/base/tableCustomSQLQuery.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/table/base/tableCustomSQLQuery.py rename to ingestion/src/metadata/data_quality/validations/table/base/tableCustomSQLQuery.py index 8e24c2a7956..579451b1d1c 100644 --- a/ingestion/src/metadata/test_suite/validations/table/base/tableCustomSQLQuery.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableCustomSQLQuery.py @@ -17,12 +17,12 @@ import traceback from abc import abstractmethod from typing import cast +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/table/base/tableRowCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/table/base/tableRowCountToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py index f63df970738..deccc52575f 100644 --- a/ingestion/src/metadata/test_suite/validations/table/base/tableRowCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py @@ -16,13 +16,13 @@ Validator for table row count to be between test case import traceback from abc import abstractmethod +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/table/base/tableRowCountToEqual.py b/ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToEqual.py similarity index 96% rename from ingestion/src/metadata/test_suite/validations/table/base/tableRowCountToEqual.py rename to ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToEqual.py index 6e3956b2a3e..e8f468a1f4b 100644 --- a/ingestion/src/metadata/test_suite/validations/table/base/tableRowCountToEqual.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToEqual.py @@ -16,13 +16,13 @@ Validator for table row count to equal test case import traceback from abc import abstractmethod +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/table/base/tableRowInsertedCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/base/tableRowInsertedCountToBeBetween.py similarity index 97% rename from ingestion/src/metadata/test_suite/validations/table/base/tableRowInsertedCountToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/table/base/tableRowInsertedCountToBeBetween.py index 95176acae26..cc88ed7af76 100644 --- a/ingestion/src/metadata/test_suite/validations/table/base/tableRowInsertedCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableRowInsertedCountToBeBetween.py @@ -17,12 +17,12 @@ import traceback from abc import abstractmethod from typing import cast +from metadata.data_quality.validations.base_test_handler import BaseTestValidator from metadata.generated.schema.tests.basic import ( TestCaseResult, TestCaseStatus, TestResultValue, ) -from metadata.test_suite.validations.base_test_handler import BaseTestValidator from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/table/pandas/__init__.py b/ingestion/src/metadata/data_quality/validations/table/pandas/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/validations/table/pandas/__init__.py rename to ingestion/src/metadata/data_quality/validations/table/pandas/__init__.py diff --git a/ingestion/src/metadata/test_suite/validations/table/pandas/tableColumnCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/pandas/tableColumnCountToBeBetween.py similarity index 86% rename from ingestion/src/metadata/test_suite/validations/table/pandas/tableColumnCountToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/table/pandas/tableColumnCountToBeBetween.py index cbc7adf02ce..1606756421b 100644 --- a/ingestion/src/metadata/test_suite/validations/table/pandas/tableColumnCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/pandas/tableColumnCountToBeBetween.py @@ -15,10 +15,10 @@ Validator for table column count to be between test case from typing import Optional -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) -from metadata.test_suite.validations.table.base.tableColumnCountToBeBetween import ( +from metadata.data_quality.validations.table.base.tableColumnCountToBeBetween import ( BaseTableColumnCountToBeBetweenValidator, ) diff --git a/ingestion/src/metadata/test_suite/validations/table/pandas/tableColumnCountToEqual.py b/ingestion/src/metadata/data_quality/validations/table/pandas/tableColumnCountToEqual.py similarity index 87% rename from ingestion/src/metadata/test_suite/validations/table/pandas/tableColumnCountToEqual.py rename to ingestion/src/metadata/data_quality/validations/table/pandas/tableColumnCountToEqual.py index ae3d1863e67..ba9d8fe481f 100644 --- a/ingestion/src/metadata/test_suite/validations/table/pandas/tableColumnCountToEqual.py +++ b/ingestion/src/metadata/data_quality/validations/table/pandas/tableColumnCountToEqual.py @@ -15,10 +15,10 @@ Validator for table column count to be equal test case from typing import Optional -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) -from metadata.test_suite.validations.table.base.tableColumnCountToEqual import ( +from metadata.data_quality.validations.table.base.tableColumnCountToEqual import ( BaseTableColumnCountToEqualValidator, ) from metadata.utils.logger import test_suite_logger diff --git a/ingestion/src/metadata/test_suite/validations/table/pandas/tableColumnNameToExist.py b/ingestion/src/metadata/data_quality/validations/table/pandas/tableColumnNameToExist.py similarity index 88% rename from ingestion/src/metadata/test_suite/validations/table/pandas/tableColumnNameToExist.py rename to ingestion/src/metadata/data_quality/validations/table/pandas/tableColumnNameToExist.py index ddf5a4042dc..3de541bce99 100644 --- a/ingestion/src/metadata/test_suite/validations/table/pandas/tableColumnNameToExist.py +++ b/ingestion/src/metadata/data_quality/validations/table/pandas/tableColumnNameToExist.py @@ -13,10 +13,10 @@ Validator for table column name to exist test case """ -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) -from metadata.test_suite.validations.table.base.tableColumnNameToExist import ( +from metadata.data_quality.validations.table.base.tableColumnNameToExist import ( BaseTableColumnNameToExistValidator, ) from metadata.utils.logger import test_suite_logger diff --git a/ingestion/src/metadata/test_suite/validations/table/pandas/tableColumnToMatchSet.py b/ingestion/src/metadata/data_quality/validations/table/pandas/tableColumnToMatchSet.py similarity index 88% rename from ingestion/src/metadata/test_suite/validations/table/pandas/tableColumnToMatchSet.py rename to ingestion/src/metadata/data_quality/validations/table/pandas/tableColumnToMatchSet.py index faee1057c48..918bbc6d614 100644 --- a/ingestion/src/metadata/test_suite/validations/table/pandas/tableColumnToMatchSet.py +++ b/ingestion/src/metadata/data_quality/validations/table/pandas/tableColumnToMatchSet.py @@ -14,10 +14,10 @@ Validator for table column name to match set test case """ -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) -from metadata.test_suite.validations.table.base.tableColumnToMatchSet import ( +from metadata.data_quality.validations.table.base.tableColumnToMatchSet import ( BaseTableColumnToMatchSetValidator, ) from metadata.utils.logger import test_suite_logger diff --git a/ingestion/src/metadata/test_suite/validations/table/pandas/tableCustomSQLQuery.py b/ingestion/src/metadata/data_quality/validations/table/pandas/tableCustomSQLQuery.py similarity index 86% rename from ingestion/src/metadata/test_suite/validations/table/pandas/tableCustomSQLQuery.py rename to ingestion/src/metadata/data_quality/validations/table/pandas/tableCustomSQLQuery.py index bd782bb432c..6193328d4d1 100644 --- a/ingestion/src/metadata/test_suite/validations/table/pandas/tableCustomSQLQuery.py +++ b/ingestion/src/metadata/data_quality/validations/table/pandas/tableCustomSQLQuery.py @@ -13,10 +13,10 @@ Validator for table custom SQL Query test case """ -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) -from metadata.test_suite.validations.table.base.tableCustomSQLQuery import ( +from metadata.data_quality.validations.table.base.tableCustomSQLQuery import ( BaseTableCustomSQLQueryValidator, ) diff --git a/ingestion/src/metadata/test_suite/validations/table/pandas/tableRowCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/pandas/tableRowCountToBeBetween.py similarity index 87% rename from ingestion/src/metadata/test_suite/validations/table/pandas/tableRowCountToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/table/pandas/tableRowCountToBeBetween.py index 420ba9f2c70..3440e50ac3e 100644 --- a/ingestion/src/metadata/test_suite/validations/table/pandas/tableRowCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/pandas/tableRowCountToBeBetween.py @@ -15,13 +15,13 @@ Validator for table row count to be between test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) -from metadata.test_suite.validations.table.base.tableRowCountToBeBetween import ( +from metadata.data_quality.validations.table.base.tableRowCountToBeBetween import ( BaseTableRowCountToBeBetweenValidator, ) +from metadata.profiler.metrics.registry import Metrics class TableRowCountToBeBetweenValidator( diff --git a/ingestion/src/metadata/test_suite/validations/table/pandas/tableRowCountToEqual.py b/ingestion/src/metadata/data_quality/validations/table/pandas/tableRowCountToEqual.py similarity index 87% rename from ingestion/src/metadata/test_suite/validations/table/pandas/tableRowCountToEqual.py rename to ingestion/src/metadata/data_quality/validations/table/pandas/tableRowCountToEqual.py index 043fc0578f8..7bd6c0049c2 100644 --- a/ingestion/src/metadata/test_suite/validations/table/pandas/tableRowCountToEqual.py +++ b/ingestion/src/metadata/data_quality/validations/table/pandas/tableRowCountToEqual.py @@ -15,13 +15,13 @@ Validator for table row count to be equal test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) -from metadata.test_suite.validations.table.base.tableRowCountToEqual import ( +from metadata.data_quality.validations.table.base.tableRowCountToEqual import ( BaseTableRowCountToEqualValidator, ) +from metadata.profiler.metrics.registry import Metrics class TableRowCountToEqualValidator( diff --git a/ingestion/src/metadata/test_suite/validations/table/pandas/tableRowInsertedCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py similarity index 94% rename from ingestion/src/metadata/test_suite/validations/table/pandas/tableRowInsertedCountToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py index 8718ebd2e23..bec8ae6fcff 100644 --- a/ingestion/src/metadata/test_suite/validations/table/pandas/tableRowInsertedCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py @@ -17,10 +17,10 @@ from datetime import datetime from dateutil.relativedelta import relativedelta -from metadata.test_suite.validations.mixins.pandas_validator_mixin import ( +from metadata.data_quality.validations.mixins.pandas_validator_mixin import ( PandasValidatorMixin, ) -from metadata.test_suite.validations.table.base.tableRowInsertedCountToBeBetween import ( +from metadata.data_quality.validations.table.base.tableRowInsertedCountToBeBetween import ( BaseTableRowInsertedCountToBeBetweenValidator, ) diff --git a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/__init__.py b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/validations/table/sqlalchemy/__init__.py rename to ingestion/src/metadata/data_quality/validations/table/sqlalchemy/__init__.py diff --git a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableColumnCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableColumnCountToBeBetween.py similarity index 86% rename from ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableColumnCountToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableColumnCountToBeBetween.py index f740a715170..6bcee7fc5b9 100644 --- a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableColumnCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableColumnCountToBeBetween.py @@ -17,8 +17,10 @@ from typing import Optional from sqlalchemy import inspect -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin -from metadata.test_suite.validations.table.base.tableColumnCountToBeBetween import ( +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.data_quality.validations.table.base.tableColumnCountToBeBetween import ( BaseTableColumnCountToBeBetweenValidator, ) diff --git a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableColumnCountToEqual.py b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableColumnCountToEqual.py similarity index 86% rename from ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableColumnCountToEqual.py rename to ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableColumnCountToEqual.py index d7628f17f5a..5d4795aea9b 100644 --- a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableColumnCountToEqual.py +++ b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableColumnCountToEqual.py @@ -17,8 +17,10 @@ from typing import Optional from sqlalchemy import inspect -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin -from metadata.test_suite.validations.table.base.tableColumnCountToEqual import ( +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.data_quality.validations.table.base.tableColumnCountToEqual import ( BaseTableColumnCountToEqualValidator, ) diff --git a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableColumnNameToExist.py b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableColumnNameToExist.py similarity index 87% rename from ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableColumnNameToExist.py rename to ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableColumnNameToExist.py index 732b4f0b3cf..e18f15da255 100644 --- a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableColumnNameToExist.py +++ b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableColumnNameToExist.py @@ -15,8 +15,10 @@ Validator for table column nanme to exist test case from sqlalchemy import inspect -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin -from metadata.test_suite.validations.table.base.tableColumnNameToExist import ( +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.data_quality.validations.table.base.tableColumnNameToExist import ( BaseTableColumnNameToExistValidator, ) from metadata.utils.logger import test_suite_logger diff --git a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableColumnToMatchSet.py b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableColumnToMatchSet.py similarity index 87% rename from ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableColumnToMatchSet.py rename to ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableColumnToMatchSet.py index 8b29d73bee1..32743f2d441 100644 --- a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableColumnToMatchSet.py +++ b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableColumnToMatchSet.py @@ -18,8 +18,10 @@ from typing import Optional from sqlalchemy import inspect -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin -from metadata.test_suite.validations.table.base.tableColumnToMatchSet import ( +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.data_quality.validations.table.base.tableColumnToMatchSet import ( BaseTableColumnToMatchSetValidator, ) from metadata.utils.logger import test_suite_logger diff --git a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableCustomSQLQuery.py b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableCustomSQLQuery.py similarity index 85% rename from ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableCustomSQLQuery.py rename to ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableCustomSQLQuery.py index cb7e7e2949c..af660ea54c2 100644 --- a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableCustomSQLQuery.py +++ b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableCustomSQLQuery.py @@ -15,8 +15,10 @@ Validator for table custom SQL Query test case from sqlalchemy import text -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin -from metadata.test_suite.validations.table.base.tableCustomSQLQuery import ( +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.data_quality.validations.table.base.tableCustomSQLQuery import ( BaseTableCustomSQLQueryValidator, ) diff --git a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableRowCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableRowCountToBeBetween.py similarity index 86% rename from ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableRowCountToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableRowCountToBeBetween.py index 3743ea7deb5..c0a934ec5ad 100644 --- a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableRowCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableRowCountToBeBetween.py @@ -15,11 +15,13 @@ Validator for table row count to be between test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin -from metadata.test_suite.validations.table.base.tableRowCountToBeBetween import ( +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.data_quality.validations.table.base.tableRowCountToBeBetween import ( BaseTableRowCountToBeBetweenValidator, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.logger import test_suite_logger logger = test_suite_logger() diff --git a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableRowCountToEqual.py b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableRowCountToEqual.py similarity index 86% rename from ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableRowCountToEqual.py rename to ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableRowCountToEqual.py index d7abc5f8848..bed0449341a 100644 --- a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableRowCountToEqual.py +++ b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableRowCountToEqual.py @@ -15,11 +15,13 @@ Validator for table row inserted count to be between test case from typing import Optional -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin -from metadata.test_suite.validations.table.base.tableRowCountToEqual import ( +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.data_quality.validations.table.base.tableRowCountToEqual import ( BaseTableRowCountToEqualValidator, ) +from metadata.profiler.metrics.registry import Metrics class TableRowCountToEqualValidator( diff --git a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py similarity index 92% rename from ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py rename to ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py index 43e47772ef4..f3c2752d7ff 100644 --- a/ingestion/src/metadata/test_suite/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py @@ -15,11 +15,13 @@ Validator for table row inserted count to be between test case from sqlalchemy import Column, text -from metadata.profiler.metrics.registry import Metrics -from metadata.test_suite.validations.mixins.sqa_validator_mixin import SQAValidatorMixin -from metadata.test_suite.validations.table.base.tableRowInsertedCountToBeBetween import ( +from metadata.data_quality.validations.mixins.sqa_validator_mixin import ( + SQAValidatorMixin, +) +from metadata.data_quality.validations.table.base.tableRowInsertedCountToBeBetween import ( BaseTableRowInsertedCountToBeBetweenValidator, ) +from metadata.profiler.metrics.registry import Metrics from metadata.utils.sqa_utils import ( dispatch_to_date_or_datetime, get_partition_col_type, diff --git a/ingestion/src/metadata/test_suite/validations/validator.py b/ingestion/src/metadata/data_quality/validations/validator.py similarity index 100% rename from ingestion/src/metadata/test_suite/validations/validator.py rename to ingestion/src/metadata/data_quality/validations/validator.py diff --git a/ingestion/src/metadata/test_suite/runner/__init__.py b/ingestion/src/metadata/mixins/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/runner/__init__.py rename to ingestion/src/metadata/mixins/__init__.py diff --git a/ingestion/src/metadata/test_suite/sink/__init__.py b/ingestion/src/metadata/mixins/pandas/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/sink/__init__.py rename to ingestion/src/metadata/mixins/pandas/__init__.py diff --git a/ingestion/src/metadata/interfaces/datalake/mixins/pandas_mixin.py b/ingestion/src/metadata/mixins/pandas/pandas_mixin.py similarity index 95% rename from ingestion/src/metadata/interfaces/datalake/mixins/pandas_mixin.py rename to ingestion/src/metadata/mixins/pandas/pandas_mixin.py index 3835bddb66e..94d85187f84 100644 --- a/ingestion/src/metadata/interfaces/datalake/mixins/pandas_mixin.py +++ b/ingestion/src/metadata/mixins/pandas/pandas_mixin.py @@ -13,10 +13,10 @@ Interfaces with database for all database engine supporting sqlalchemy abstraction layer """ -from metadata.generated.schema.entity.data.table import PartitionIntervalType -from metadata.test_suite.validations.table.pandas.tableRowInsertedCountToBeBetween import ( +from metadata.data_quality.validations.table.pandas.tableRowInsertedCountToBeBetween import ( TableRowInsertedCountToBeBetweenValidator, ) +from metadata.generated.schema.entity.data.table import PartitionIntervalType class PandasInterfaceMixin: diff --git a/ingestion/src/metadata/test_suite/validations/mixins/__init__.py b/ingestion/src/metadata/mixins/sqalchemy/__init__.py similarity index 100% rename from ingestion/src/metadata/test_suite/validations/mixins/__init__.py rename to ingestion/src/metadata/mixins/sqalchemy/__init__.py diff --git a/ingestion/src/metadata/interfaces/sqalchemy/mixins/sqa_mixin.py b/ingestion/src/metadata/mixins/sqalchemy/sqa_mixin.py similarity index 100% rename from ingestion/src/metadata/interfaces/sqalchemy/mixins/sqa_mixin.py rename to ingestion/src/metadata/mixins/sqalchemy/sqa_mixin.py diff --git a/ingestion/src/metadata/profiler/api/models.py b/ingestion/src/metadata/profiler/api/models.py index 81a4643f9fa..02bcc706247 100644 --- a/ingestion/src/metadata/profiler/api/models.py +++ b/ingestion/src/metadata/profiler/api/models.py @@ -29,7 +29,7 @@ from metadata.generated.schema.entity.data.table import ( TableData, ) from metadata.generated.schema.type.basic import FullyQualifiedEntityName -from metadata.profiler.profiler.models import ProfilerDef +from metadata.profiler.processor.models import ProfilerDef class ColumnConfig(ConfigModel): diff --git a/ingestion/src/metadata/profiler/api/workflow.py b/ingestion/src/metadata/profiler/api/workflow.py index 5aaace55a28..cbdd6084568 100644 --- a/ingestion/src/metadata/profiler/api/workflow.py +++ b/ingestion/src/metadata/profiler/api/workflow.py @@ -59,16 +59,16 @@ from metadata.profiler.api.models import ( ProfilerResponse, TableConfig, ) -from metadata.profiler.metrics.registry import Metrics -from metadata.profiler.profiler.core import Profiler -from metadata.profiler.profiler.default import DefaultProfiler, get_default_metrics -from metadata.profiler.profiler.interface.pandas.pandas_profiler_interface import ( +from metadata.profiler.interface.pandas.pandas_profiler_interface import ( PandasProfilerInterface, ) -from metadata.profiler.profiler.interface.profiler_protocol import ProfilerProtocol -from metadata.profiler.profiler.interface.sqlalchemy.sqa_profiler_interface import ( +from metadata.profiler.interface.profiler_protocol import ProfilerProtocol +from metadata.profiler.interface.sqlalchemy.sqa_profiler_interface import ( SQAProfilerInterface, ) +from metadata.profiler.metrics.registry import Metrics +from metadata.profiler.processor.core import Profiler +from metadata.profiler.processor.default import DefaultProfiler, get_default_metrics from metadata.utils import fqn from metadata.utils.class_helper import ( get_service_class_from_service_type, diff --git a/ingestion/src/metadata/profiler/profiler/interface/pandas/pandas_profiler_interface.py b/ingestion/src/metadata/profiler/interface/pandas/pandas_profiler_interface.py similarity index 98% rename from ingestion/src/metadata/profiler/profiler/interface/pandas/pandas_profiler_interface.py rename to ingestion/src/metadata/profiler/interface/pandas/pandas_profiler_interface.py index 32dfd42bddb..853f1c46d87 100644 --- a/ingestion/src/metadata/profiler/profiler/interface/pandas/pandas_profiler_interface.py +++ b/ingestion/src/metadata/profiler/interface/pandas/pandas_profiler_interface.py @@ -31,11 +31,11 @@ from metadata.ingestion.source.database.datalake.metadata import ( DATALAKE_DATA_TYPES, ometa_to_dataframe, ) -from metadata.interfaces.datalake.mixins.pandas_mixin import PandasInterfaceMixin +from metadata.mixins.pandas.pandas_mixin import PandasInterfaceMixin +from metadata.profiler.interface.profiler_protocol import ProfilerProtocol from metadata.profiler.metrics.core import MetricTypes from metadata.profiler.metrics.registry import Metrics -from metadata.profiler.profiler.datalake_sampler import DatalakeSampler -from metadata.profiler.profiler.interface.profiler_protocol import ProfilerProtocol +from metadata.profiler.processor.datalake_sampler import DatalakeSampler from metadata.utils.dispatch import valuedispatch from metadata.utils.logger import profiler_interface_registry_logger from metadata.utils.sqa_like_column import SQALikeColumn, Type diff --git a/ingestion/src/metadata/profiler/profiler/interface/profiler_protocol.py b/ingestion/src/metadata/profiler/interface/profiler_protocol.py similarity index 100% rename from ingestion/src/metadata/profiler/profiler/interface/profiler_protocol.py rename to ingestion/src/metadata/profiler/interface/profiler_protocol.py diff --git a/ingestion/src/metadata/profiler/profiler/interface/sqlalchemy/sqa_profiler_interface.py b/ingestion/src/metadata/profiler/interface/sqlalchemy/sqa_profiler_interface.py similarity index 98% rename from ingestion/src/metadata/profiler/profiler/interface/sqlalchemy/sqa_profiler_interface.py rename to ingestion/src/metadata/profiler/interface/sqlalchemy/sqa_profiler_interface.py index fb561cb6c61..78d22f8406a 100644 --- a/ingestion/src/metadata/profiler/profiler/interface/sqlalchemy/sqa_profiler_interface.py +++ b/ingestion/src/metadata/profiler/interface/sqlalchemy/sqa_profiler_interface.py @@ -30,15 +30,15 @@ from metadata.generated.schema.entity.services.databaseService import DatabaseCo from metadata.ingestion.api.processor import ProfilerProcessorStatus from metadata.ingestion.connections.session import create_and_bind_thread_safe_session from metadata.ingestion.source.connections import get_connection -from metadata.interfaces.sqalchemy.mixins.sqa_mixin import SQAInterfaceMixin +from metadata.mixins.sqalchemy.sqa_mixin import SQAInterfaceMixin +from metadata.profiler.interface.profiler_protocol import ProfilerProtocol from metadata.profiler.metrics.core import MetricTypes from metadata.profiler.metrics.registry import Metrics from metadata.profiler.metrics.static.mean import Mean from metadata.profiler.metrics.static.stddev import StdDev from metadata.profiler.metrics.static.sum import Sum -from metadata.profiler.profiler.interface.profiler_protocol import ProfilerProtocol -from metadata.profiler.profiler.runner import QueryRunner -from metadata.profiler.profiler.sampler import Sampler +from metadata.profiler.processor.runner import QueryRunner +from metadata.profiler.processor.sampler import Sampler from metadata.utils.custom_thread_pool import CustomThreadPoolExecutor from metadata.utils.dispatch import valuedispatch from metadata.utils.logger import profiler_interface_registry_logger diff --git a/ingestion/src/metadata/profiler/profiler/README.md b/ingestion/src/metadata/profiler/processor/README.md similarity index 100% rename from ingestion/src/metadata/profiler/profiler/README.md rename to ingestion/src/metadata/profiler/processor/README.md diff --git a/ingestion/src/metadata/profiler/processor/__init__.py b/ingestion/src/metadata/profiler/processor/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/src/metadata/profiler/profiler/core.py b/ingestion/src/metadata/profiler/processor/core.py similarity index 99% rename from ingestion/src/metadata/profiler/profiler/core.py rename to ingestion/src/metadata/profiler/processor/core.py index e7e60c3b520..c6b241baed5 100644 --- a/ingestion/src/metadata/profiler/profiler/core.py +++ b/ingestion/src/metadata/profiler/processor/core.py @@ -35,6 +35,7 @@ from metadata.generated.schema.entity.data.table import ( ) from metadata.ingestion.processor.pii import NERScanner from metadata.profiler.api.models import ProfilerResponse +from metadata.profiler.interface.profiler_protocol import ProfilerProtocol from metadata.profiler.metrics.core import ( ComposedMetric, CustomMetric, @@ -48,7 +49,6 @@ from metadata.profiler.metrics.core import ( from metadata.profiler.metrics.registry import Metrics from metadata.profiler.metrics.static.row_count import RowCount from metadata.profiler.orm.registry import NOT_COMPUTE -from metadata.profiler.profiler.interface.profiler_protocol import ProfilerProtocol from metadata.utils.logger import profiler_logger logger = profiler_logger() diff --git a/ingestion/src/metadata/profiler/profiler/datalake_sampler.py b/ingestion/src/metadata/profiler/processor/datalake_sampler.py similarity index 100% rename from ingestion/src/metadata/profiler/profiler/datalake_sampler.py rename to ingestion/src/metadata/profiler/processor/datalake_sampler.py diff --git a/ingestion/src/metadata/profiler/profiler/default.py b/ingestion/src/metadata/profiler/processor/default.py similarity index 94% rename from ingestion/src/metadata/profiler/profiler/default.py rename to ingestion/src/metadata/profiler/processor/default.py index a6714c114fd..19bea61c246 100644 --- a/ingestion/src/metadata/profiler/profiler/default.py +++ b/ingestion/src/metadata/profiler/processor/default.py @@ -17,10 +17,10 @@ from typing import List, Optional from sqlalchemy.orm import DeclarativeMeta from metadata.generated.schema.entity.data.table import ColumnProfilerConfig +from metadata.profiler.interface.profiler_protocol import ProfilerProtocol from metadata.profiler.metrics.core import Metric, add_props from metadata.profiler.metrics.registry import Metrics -from metadata.profiler.profiler.core import Profiler -from metadata.profiler.profiler.interface.profiler_protocol import ProfilerProtocol +from metadata.profiler.processor.core import Profiler def get_default_metrics(table: DeclarativeMeta) -> List[Metric]: diff --git a/ingestion/src/metadata/profiler/profiler/handle_partition.py b/ingestion/src/metadata/profiler/processor/handle_partition.py similarity index 100% rename from ingestion/src/metadata/profiler/profiler/handle_partition.py rename to ingestion/src/metadata/profiler/processor/handle_partition.py diff --git a/ingestion/src/metadata/profiler/profiler/models.py b/ingestion/src/metadata/profiler/processor/models.py similarity index 100% rename from ingestion/src/metadata/profiler/profiler/models.py rename to ingestion/src/metadata/profiler/processor/models.py diff --git a/ingestion/src/metadata/profiler/profiler/runner.py b/ingestion/src/metadata/profiler/processor/runner.py similarity index 98% rename from ingestion/src/metadata/profiler/profiler/runner.py rename to ingestion/src/metadata/profiler/processor/runner.py index d343bd59501..3d2deb7d7bc 100644 --- a/ingestion/src/metadata/profiler/profiler/runner.py +++ b/ingestion/src/metadata/profiler/processor/runner.py @@ -22,7 +22,7 @@ from sqlalchemy import text from sqlalchemy.orm import DeclarativeMeta, Query, Session from sqlalchemy.orm.util import AliasedClass -from metadata.profiler.profiler.handle_partition import partition_filter_handler +from metadata.profiler.processor.handle_partition import partition_filter_handler from metadata.utils.logger import query_runner_logger from metadata.utils.sqa_utils import get_query_filter_for_runner diff --git a/ingestion/src/metadata/profiler/profiler/sampler.py b/ingestion/src/metadata/profiler/processor/sampler.py similarity index 99% rename from ingestion/src/metadata/profiler/profiler/sampler.py rename to ingestion/src/metadata/profiler/processor/sampler.py index 54e4f697bc3..21f0102d703 100644 --- a/ingestion/src/metadata/profiler/profiler/sampler.py +++ b/ingestion/src/metadata/profiler/processor/sampler.py @@ -29,7 +29,7 @@ from metadata.profiler.api.models import ProfileSampleConfig from metadata.profiler.orm.functions.modulo import ModuloFn from metadata.profiler.orm.functions.random_num import RandomNumFn from metadata.profiler.orm.registry import Dialects -from metadata.profiler.profiler.handle_partition import partition_filter_handler +from metadata.profiler.processor.handle_partition import partition_filter_handler from metadata.utils.sqa_utils import ( build_query_filter, dispatch_to_date_or_datetime, diff --git a/ingestion/src/metadata/utils/importer.py b/ingestion/src/metadata/utils/importer.py index e4df75ac150..818f085b193 100644 --- a/ingestion/src/metadata/utils/importer.py +++ b/ingestion/src/metadata/utils/importer.py @@ -231,7 +231,7 @@ def import_test_case_class( test_definition[0].upper() + test_definition[1:] ) # change test names to camel case return import_from_module( - "metadata.test_suite.validations.{}.{}.{}.{}Validator".format( + "metadata.data_quality.validations.{}.{}.{}.{}Validator".format( test_type.lower(), runner_type, test_definition, diff --git a/ingestion/tests/integration/test_suite/test_e2e_workflow.py b/ingestion/tests/integration/test_suite/test_e2e_workflow.py index 81580d5e4a7..a03436d1fa7 100644 --- a/ingestion/tests/integration/test_suite/test_e2e_workflow.py +++ b/ingestion/tests/integration/test_suite/test_e2e_workflow.py @@ -20,6 +20,7 @@ from datetime import datetime, timedelta import sqlalchemy as sqa from sqlalchemy.orm import Session, declarative_base +from metadata.data_quality.api.workflow import TestSuiteWorkflow from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest from metadata.generated.schema.api.data.createDatabaseSchema import ( CreateDatabaseSchemaRequest, @@ -45,7 +46,6 @@ from metadata.generated.schema.entity.services.databaseService import ( ) from metadata.generated.schema.tests.testCase import TestCase from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.test_suite.api.workflow import TestSuiteWorkflow test_suite_config = { "source": { diff --git a/ingestion/tests/integration/test_suite/test_registry_names_match_test_definition.py b/ingestion/tests/integration/test_suite/test_registry_names_match_test_definition.py index cd924ae844a..e9734047287 100644 --- a/ingestion/tests/integration/test_suite/test_registry_names_match_test_definition.py +++ b/ingestion/tests/integration/test_suite/test_registry_names_match_test_definition.py @@ -17,10 +17,10 @@ Validate the names in the registry match the ones of the test definition import pkgutil from unittest import TestCase -import metadata.test_suite.validations.column.pandas as pandas_column -import metadata.test_suite.validations.column.sqlalchemy as sqa_column -import metadata.test_suite.validations.table.pandas as pandas_table -import metadata.test_suite.validations.table.sqlalchemy as sqa_table +import metadata.data_quality.validations.column.pandas as pandas_column +import metadata.data_quality.validations.column.sqlalchemy as sqa_column +import metadata.data_quality.validations.table.pandas as pandas_table +import metadata.data_quality.validations.table.sqlalchemy as sqa_table from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) diff --git a/ingestion/tests/integration/test_suite/test_workflow.py b/ingestion/tests/integration/test_suite/test_workflow.py index 2723a14315c..040db662c83 100644 --- a/ingestion/tests/integration/test_suite/test_workflow.py +++ b/ingestion/tests/integration/test_suite/test_workflow.py @@ -17,6 +17,7 @@ import unittest from collections.abc import MutableSequence from copy import deepcopy +from metadata.data_quality.api.workflow import TestSuiteWorkflow from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, @@ -24,7 +25,6 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata from metadata.generated.schema.tests.testCase import TestCase from metadata.generated.schema.tests.testSuite import TestSuite from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.test_suite.api.workflow import TestSuiteWorkflow sqlite_shared = "file:cachedb?mode=memory&cache=shared&check_same_thread=False" diff --git a/ingestion/tests/unit/profiler/test_metrics.py b/ingestion/tests/unit/profiler/test_metrics.py index 6c9ef511648..9b3ac62eefb 100644 --- a/ingestion/tests/unit/profiler/test_metrics.py +++ b/ingestion/tests/unit/profiler/test_metrics.py @@ -27,13 +27,13 @@ from metadata.generated.schema.entity.services.connections.database.sqliteConnec SQLiteConnection, SQLiteScheme, ) +from metadata.profiler.interface.sqlalchemy.sqa_profiler_interface import ( + SQAProfilerInterface, +) from metadata.profiler.metrics.core import add_props from metadata.profiler.metrics.registry import Metrics from metadata.profiler.orm.functions.sum import SumFn -from metadata.profiler.profiler.core import Profiler -from metadata.profiler.profiler.interface.sqlalchemy.sqa_profiler_interface import ( - SQAProfilerInterface, -) +from metadata.profiler.processor.core import Profiler Base = declarative_base() diff --git a/ingestion/tests/unit/profiler/test_profiler.py b/ingestion/tests/unit/profiler/test_profiler.py index f1652dd3aab..68ff0910beb 100644 --- a/ingestion/tests/unit/profiler/test_profiler.py +++ b/ingestion/tests/unit/profiler/test_profiler.py @@ -41,13 +41,13 @@ from metadata.generated.schema.entity.services.connections.database.sqliteConnec SQLiteScheme, ) from metadata.ingestion.source import sqa_types -from metadata.profiler.metrics.core import add_props -from metadata.profiler.metrics.registry import Metrics -from metadata.profiler.profiler.core import MissingMetricException, Profiler -from metadata.profiler.profiler.default import DefaultProfiler -from metadata.profiler.profiler.interface.sqlalchemy.sqa_profiler_interface import ( +from metadata.profiler.interface.sqlalchemy.sqa_profiler_interface import ( SQAProfilerInterface, ) +from metadata.profiler.metrics.core import add_props +from metadata.profiler.metrics.registry import Metrics +from metadata.profiler.processor.core import MissingMetricException, Profiler +from metadata.profiler.processor.default import DefaultProfiler Base = declarative_base() diff --git a/ingestion/tests/unit/profiler/test_profiler_partitions.py b/ingestion/tests/unit/profiler/test_profiler_partitions.py index 3a3fb93881b..3ea1e5c550a 100644 --- a/ingestion/tests/unit/profiler/test_profiler_partitions.py +++ b/ingestion/tests/unit/profiler/test_profiler_partitions.py @@ -31,7 +31,7 @@ from metadata.generated.schema.entity.services.databaseService import ( ) from metadata.generated.schema.type.entityReference import EntityReference from metadata.profiler.api.workflow import ProfilerWorkflow -from metadata.profiler.profiler.interface.profiler_protocol import ProfilerProtocol +from metadata.profiler.interface.profiler_protocol import ProfilerProtocol """ Check Partitioned Table in Profiler Workflow diff --git a/ingestion/tests/unit/profiler/test_runner.py b/ingestion/tests/unit/profiler/test_runner.py index 0c5d05c3fac..eb165dabd9a 100644 --- a/ingestion/tests/unit/profiler/test_runner.py +++ b/ingestion/tests/unit/profiler/test_runner.py @@ -22,8 +22,8 @@ from sqlalchemy.orm import declarative_base from metadata.ingestion.connections.session import create_and_bind_session from metadata.profiler.api.models import ProfileSampleConfig -from metadata.profiler.profiler.runner import QueryRunner -from metadata.profiler.profiler.sampler import Sampler +from metadata.profiler.processor.runner import QueryRunner +from metadata.profiler.processor.sampler import Sampler from metadata.utils.timeout import cls_timeout Base = declarative_base() diff --git a/ingestion/tests/unit/profiler/test_sample.py b/ingestion/tests/unit/profiler/test_sample.py index 440c3e9d26f..272b72e6670 100644 --- a/ingestion/tests/unit/profiler/test_sample.py +++ b/ingestion/tests/unit/profiler/test_sample.py @@ -27,13 +27,13 @@ from metadata.generated.schema.entity.services.connections.database.sqliteConnec SQLiteScheme, ) from metadata.profiler.api.models import ProfileSampleConfig -from metadata.profiler.metrics.registry import Metrics -from metadata.profiler.orm.registry import CustomTypes -from metadata.profiler.profiler.core import Profiler -from metadata.profiler.profiler.interface.sqlalchemy.sqa_profiler_interface import ( +from metadata.profiler.interface.sqlalchemy.sqa_profiler_interface import ( SQAProfilerInterface, ) -from metadata.profiler.profiler.sampler import Sampler +from metadata.profiler.metrics.registry import Metrics +from metadata.profiler.orm.registry import CustomTypes +from metadata.profiler.processor.core import Profiler +from metadata.profiler.processor.sampler import Sampler Base = declarative_base() diff --git a/ingestion/tests/unit/profiler/test_sqa_profiler_interface.py b/ingestion/tests/unit/profiler/test_sqa_profiler_interface.py index 0744e2199fd..ce1e02674a3 100644 --- a/ingestion/tests/unit/profiler/test_sqa_profiler_interface.py +++ b/ingestion/tests/unit/profiler/test_sqa_profiler_interface.py @@ -38,6 +38,9 @@ from metadata.generated.schema.entity.services.connections.database.sqliteConnec SQLiteConnection, SQLiteScheme, ) +from metadata.profiler.interface.sqlalchemy.sqa_profiler_interface import ( + SQAProfilerInterface, +) from metadata.profiler.metrics.core import ( ComposedMetric, MetricTypes, @@ -45,10 +48,7 @@ from metadata.profiler.metrics.core import ( StaticMetric, ) from metadata.profiler.metrics.static.row_count import RowCount -from metadata.profiler.profiler.default import get_default_metrics -from metadata.profiler.profiler.interface.sqlalchemy.sqa_profiler_interface import ( - SQAProfilerInterface, -) +from metadata.profiler.processor.default import get_default_metrics class User(declarative_base()): diff --git a/ingestion/tests/unit/profiler/test_workflow.py b/ingestion/tests/unit/profiler/test_workflow.py index 9348011f9bc..d01468d27b0 100644 --- a/ingestion/tests/unit/profiler/test_workflow.py +++ b/ingestion/tests/unit/profiler/test_workflow.py @@ -37,11 +37,11 @@ from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline from metadata.generated.schema.type.entityReference import EntityReference from metadata.profiler.api.models import ProfilerProcessorConfig from metadata.profiler.api.workflow import ProfilerWorkflow -from metadata.profiler.profiler.default import DefaultProfiler -from metadata.profiler.profiler.interface.profiler_protocol import ProfilerProtocol -from metadata.profiler.profiler.interface.sqlalchemy.sqa_profiler_interface import ( +from metadata.profiler.interface.profiler_protocol import ProfilerProtocol +from metadata.profiler.interface.sqlalchemy.sqa_profiler_interface import ( SQAProfilerInterface, ) +from metadata.profiler.processor.default import DefaultProfiler TABLE = Table( id=uuid.uuid4(), diff --git a/ingestion/tests/unit/test_suite/conftest.py b/ingestion/tests/unit/test_suite/conftest.py index 592d5724955..e723d24a1d6 100644 --- a/ingestion/tests/unit/test_suite/conftest.py +++ b/ingestion/tests/unit/test_suite/conftest.py @@ -22,6 +22,9 @@ import pytest import sqlalchemy as sqa from sqlalchemy.orm import declarative_base +from metadata.data_quality.interface.sqlalchemy.sqa_test_suite_interface import ( + SQATestSuiteInterface, +) from metadata.generated.schema.entity.data.table import Column, DataType, Table from metadata.generated.schema.entity.services.connections.database.sqliteConnection import ( SQLiteConnection, @@ -29,7 +32,6 @@ from metadata.generated.schema.entity.services.connections.database.sqliteConnec ) from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue from metadata.generated.schema.type.entityReference import EntityReference -from metadata.interfaces.sqalchemy.sqa_test_suite_interface import SQATestSuiteInterface Base = declarative_base() @@ -39,6 +41,7 @@ ENTITY_LINK_FNAME = "<#E::table::service.db.users::columns::first name>" ENTITY_LINK_AGE = "<#E::table::service.db.users::columns::age>" ENTITY_LINK_NAME = "<#E::table::service.db.users::columns::name>" ENTITY_LINK_USER = "<#E::table::service.db.users>" +ENTITY_LINK_INSERTED_DATE = "<#E::table::service.db.users::columns::inserted_date>" TABLE = Table( id=uuid4(), @@ -615,3 +618,31 @@ def test_case_table_custom_sql_query_success_dl(): TestCaseParameterValue(name="sqlExpression", value="age < 0"), ], ) + + +@pytest.fixture +def test_case_column_values_to_be_between_date(): + return TestCase( + name=TEST_CASE_NAME, + entityLink=ENTITY_LINK_INSERTED_DATE, + testSuite=EntityReference(id=uuid4(), type="TestSuite"), # type: ignore + testDefinition=EntityReference(id=uuid4(), type="TestDefinition"), # type: ignore + parameterValues=[ + TestCaseParameterValue(name="minValue", value="1625127852000"), + TestCaseParameterValue(name="maxValue", value="1625127852000"), + ], + ) # type: ignore + + +@pytest.fixture +def test_case_column_values_to_be_between_datetime(): + return TestCase( + name=TEST_CASE_NAME, + entityLink=ENTITY_LINK_INSERTED_DATE, + testSuite=EntityReference(id=uuid4(), type="TestSuite"), # type: ignore + testDefinition=EntityReference(id=uuid4(), type="TestDefinition"), # type: ignore + parameterValues=[ + TestCaseParameterValue(name="minValue", value="1625127852000"), + TestCaseParameterValue(name="maxValue", value="1625171052000"), + ], + ) # type: ignore diff --git a/ingestion/tests/unit/test_suite/test_validations_databases.py b/ingestion/tests/unit/test_suite/test_validations_databases.py index 54180425f32..86145da4fde 100644 --- a/ingestion/tests/unit/test_suite/test_validations_databases.py +++ b/ingestion/tests/unit/test_suite/test_validations_databases.py @@ -14,12 +14,13 @@ Test Table and Column Tests' validate implementations. Each test should validate the Success, Failure and Aborted statuses """ -from datetime import datetime +from datetime import date, datetime +from unittest.mock import patch import pytest +from metadata.data_quality.validations.validator import Validator from metadata.generated.schema.tests.basic import TestCaseResult, TestCaseStatus -from metadata.test_suite.validations.validator import Validator from metadata.utils.importer import import_test_case_class EXECUTION_DATE = datetime.strptime("2021-07-03", "%Y-%m-%d") @@ -28,6 +29,28 @@ EXECUTION_DATE = datetime.strptime("2021-07-03", "%Y-%m-%d") @pytest.mark.parametrize( "test_case_name,test_case_type,test_type,expected", [ + ( + "test_case_column_values_to_be_between_date", + "columnValuesToBeBetween", + "COLUMN", + ( + TestCaseResult, + "2021-07-01 00:00:00", + "2021-07-01 23:59:59.999999", + TestCaseStatus.Failed, + ), + ), + ( + "test_case_column_values_to_be_between_datetime", + "columnValuesToBeBetween", + "COLUMN", + ( + TestCaseResult, + "2021-07-01 10:37:59", + "2021-07-01 10:37:59", + TestCaseStatus.Success, + ), + ), ( "test_case_column_value_length_to_be_between", "columnValueLengthsToBeBetween", @@ -239,20 +262,59 @@ def test_suite_validation_database( test_case = request.getfixturevalue(test_case_name) type_, val_1, val_2, status = expected - test_handler_obj = import_test_case_class( - test_type, - "sqlalchemy", - test_case_type, - ) + if test_case_name == "test_case_column_values_to_be_between_date": + with patch( + "metadata.data_quality.validations.column.sqlalchemy.columnValuesToBeBetween.ColumnValuesToBeBetweenValidator._run_results", + return_value=date(2021, 7, 1), + ): + test_handler_obj = import_test_case_class( + test_type, + "sqlalchemy", + test_case_type, + ) - test_handler = test_handler_obj( - create_sqlite_table, - test_case=test_case, - execution_date=EXECUTION_DATE.timestamp(), - ) + test_handler = test_handler_obj( + create_sqlite_table, + test_case=test_case, + execution_date=EXECUTION_DATE.timestamp(), + ) - validator = Validator(test_handler) - res = validator.validate() + validator = Validator(test_handler) + res = validator.validate() + elif test_case_name == "test_case_column_values_to_be_between_datetime": + with patch( + "metadata.data_quality.validations.column.sqlalchemy.columnValuesToBeBetween.ColumnValuesToBeBetweenValidator._run_results", + return_value=datetime(2021, 7, 1, 10, 37, 59), + ): + test_handler_obj = import_test_case_class( + test_type, + "sqlalchemy", + test_case_type, + ) + + test_handler = test_handler_obj( + create_sqlite_table, + test_case=test_case, + execution_date=EXECUTION_DATE.timestamp(), + ) + + validator = Validator(test_handler) + res = validator.validate() + else: + test_handler_obj = import_test_case_class( + test_type, + "sqlalchemy", + test_case_type, + ) + + test_handler = test_handler_obj( + create_sqlite_table, + test_case=test_case, + execution_date=EXECUTION_DATE.timestamp(), + ) + + validator = Validator(test_handler) + res = validator.validate() assert isinstance(res, type_) if val_1: diff --git a/ingestion/tests/unit/test_suite/test_validations_datalake.py b/ingestion/tests/unit/test_suite/test_validations_datalake.py index 27870296f90..18c3b6b2a20 100644 --- a/ingestion/tests/unit/test_suite/test_validations_datalake.py +++ b/ingestion/tests/unit/test_suite/test_validations_datalake.py @@ -20,8 +20,8 @@ from datetime import datetime, timedelta import pytest from pandas import DataFrame +from metadata.data_quality.validations.validator import Validator from metadata.generated.schema.tests.basic import TestCaseResult, TestCaseStatus -from metadata.test_suite.validations.validator import Validator from metadata.utils.importer import import_test_case_class EXECUTION_DATE = datetime.strptime("2021-07-03", "%Y-%m-%d") diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/test_suite.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/test_suite.py index e43e8909d2f..5246e9ee8a7 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/test_suite.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/test_suite.py @@ -17,6 +17,7 @@ from airflow import DAG from openmetadata_managed_apis.utils.logger import set_operator_logger from openmetadata_managed_apis.workflows.ingestion.common import build_dag, build_source +from metadata.data_quality.api.workflow import TestSuiteWorkflow from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import ( IngestionPipeline, ) @@ -28,7 +29,6 @@ from metadata.generated.schema.metadataIngestion.workflow import ( WorkflowConfig, ) from metadata.ingestion.models.encoders import show_secrets_encoder -from metadata.test_suite.api.workflow import TestSuiteWorkflow def test_suite_workflow(workflow_config: OpenMetadataWorkflowConfig): diff --git a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py index 2c7d0c1dd81..1edc4907bcf 100644 --- a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py +++ b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py @@ -33,6 +33,7 @@ from openmetadata_managed_apis.workflows.ingestion.usage import ( build_usage_workflow_config, ) +from metadata.data_quality.api.workflow import TestSuiteWorkflow from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, @@ -72,7 +73,6 @@ from metadata.ingestion.api.workflow import Workflow from metadata.ingestion.models.encoders import show_secrets_encoder from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.profiler.api.workflow import ProfilerWorkflow -from metadata.test_suite.api.workflow import TestSuiteWorkflow def mock_set_ingestion_pipeline_status(self, state): diff --git a/openmetadata-docs/content/connectors/ingestion/workflows/data-quality/index.md b/openmetadata-docs/content/connectors/ingestion/workflows/data-quality/index.md index 834e3c0a913..c459f796e4d 100644 --- a/openmetadata-docs/content/connectors/ingestion/workflows/data-quality/index.md +++ b/openmetadata-docs/content/connectors/ingestion/workflows/data-quality/index.md @@ -187,7 +187,7 @@ except ModuleNotFoundError: from airflow.operators.python_operator import PythonOperator from metadata.config.common import load_config_file -from metadata.test_suite.api.workflow import TestSuiteWorkflow +from metadata.data_quality.api.workflow import TestSuiteWorkflow from airflow.utils.dates import days_ago default_args = {