From 7863f040d1486d7920d0e4293631d973d66fd7b2 Mon Sep 17 00:00:00 2001 From: Nahuel Date: Thu, 25 Aug 2022 11:19:31 +0200 Subject: [PATCH] Fix#6027: Improve logging in ORM profiler (#6919) * Improve logging in ORM profiler * Fix failing test * Updated logging in test_suite module to match company's format Co-authored-by: Teddy Crepineau --- .../src/metadata/interfaces/sqa_interface.py | 44 +++++++++++-------- .../src/metadata/orm_profiler/api/workflow.py | 27 ++++++++---- .../metrics/static/count_in_set.py | 7 +-- .../metadata/orm_profiler/orm/types/uuid.py | 5 ++- .../metadata/orm_profiler/profiler/core.py | 4 +- .../orm_profiler/sink/metadata_rest.py | 6 +-- .../src/metadata/test_suite/api/workflow.py | 11 +++-- .../column/column_value_max_to_be_between.py | 7 +-- .../column/column_value_mean_to_be_between.py | 6 ++- .../column_value_median_to_be_between.py | 6 ++- .../column/column_value_min_to_be_between.py | 6 ++- .../column_value_stddev_to_be_between.py | 9 ++-- .../column/column_values_in_set.py | 10 ++--- .../column_values_length_to_be_between.py | 6 ++- ...column_values_missing_count_to_be_equal.py | 17 ++++--- .../column/column_values_not_in_set.py | 7 +-- .../column/column_values_sum_to_be_between.py | 6 ++- .../column/column_values_to_be_between.py | 9 ++-- .../column/column_values_to_be_not_null.py | 9 ++-- .../column/column_values_to_be_unique.py | 9 ++-- .../column/column_values_to_match_regex.py | 10 ++--- .../column_values_to_not_match_regex.py | 7 +-- .../table/table_column_count_to_be_between.py | 6 ++- .../table/table_column_count_to_equal.py | 6 ++- .../table/table_column_name_to_exist.py | 6 ++- .../table/table_column_to_match_set.py | 6 ++- .../table/table_custom_sql_query.py | 6 ++- .../table/table_row_count_to_be_between.py | 6 ++- .../table/table_row_count_to_equal.py | 6 ++- 29 files changed, 154 insertions(+), 116 deletions(-) diff --git a/ingestion/src/metadata/interfaces/sqa_interface.py b/ingestion/src/metadata/interfaces/sqa_interface.py index 482e0bbd448..beecd433b49 100644 --- a/ingestion/src/metadata/interfaces/sqa_interface.py +++ b/ingestion/src/metadata/interfaces/sqa_interface.py @@ -16,6 +16,7 @@ supporting sqlalchemy abstraction layer import concurrent.futures import threading +import traceback from collections import defaultdict from datetime import datetime, timezone from typing import Dict, List, Optional, Union @@ -164,9 +165,10 @@ class SQAInterface(InterfaceProtocol): try: self._metadata = OpenMetadata(metadata_config) self._metadata.health_check() - except Exception: + except Exception as exc: + logger.debug(traceback.format_exc()) logger.warning( - "No OpenMetadata server configuration found. Running profiler interface without OM server connection" + f"No OpenMetadata server configuration found. Running profiler interface without OM server connection: {exc}" ) self._metadata = None @@ -248,8 +250,9 @@ class SQAInterface(InterfaceProtocol): try: column = column.name - except Exception as err: - logger.debug(err) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Unexpected exception computing metrics: {exc}") return row, column @@ -331,8 +334,9 @@ class SQAInterface(InterfaceProtocol): """ try: return metric(column).fn(column_results) - except Exception as err: - logger.error(err) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Unexpected exception computing metrics: {exc}") self.session.rollback() def run_test_case( @@ -382,9 +386,10 @@ def get_table_metrics( if row: return dict(row) - except Exception as err: - logger.error( - f"Error trying to compute profile for {runner.table.__tablename__} - {err}" + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning( + f"Error trying to compute profile for {runner.table.__tablename__}: {exc}" ) session.rollback() @@ -415,9 +420,10 @@ def get_static_metrics( ] ) return dict(row) - except Exception as err: - logger.error( - f"Error trying to compute profile for {runner.table.__tablename__}.{column.name} - {err}" + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning( + f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}" ) session.rollback() @@ -453,9 +459,10 @@ def get_query_metrics( else: row = runner.select_first_from_query(metric_query) return dict(row) - except Exception as err: - logger.error( - f"Error trying to compute profile for {runner.table.__tablename__}.{column.name} - {err}" + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning( + f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}" ) session.rollback() @@ -482,9 +489,10 @@ def get_window_metrics( if not isinstance(row, Row): return {metric.name(): row} return dict(row) - except Exception as err: - logger.error( - f"Error trying to compute profile for {runner.table.__tablename__}.{column.name} - {err}" + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning( + f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}" ) session.rollback() diff --git a/ingestion/src/metadata/orm_profiler/api/workflow.py b/ingestion/src/metadata/orm_profiler/api/workflow.py index e3b26c3be9a..c2f0ceabfb3 100644 --- a/ingestion/src/metadata/orm_profiler/api/workflow.py +++ b/ingestion/src/metadata/orm_profiler/api/workflow.py @@ -120,7 +120,10 @@ class ProfilerWorkflow: config = parse_workflow_config_gracefully(config_dict) return cls(config) except ValidationError as err: - logger.error("Error trying to parse the Profiler Workflow configuration") + logger.debug(traceback.format_exc()) + logger.error( + f"Error trying to parse the Profiler Workflow configuration: {err}" + ) raise err def get_config_for_entity(self, entity: Table) -> Optional[TableConfig]: @@ -293,10 +296,12 @@ class ProfilerWorkflow: self.source_status.scanned(table.fullyQualifiedName.__root__) yield table - except Exception as err: # pylint: disable=broad-except - self.source_status.filter(table.fullyQualifiedName.__root__, f"{err}") - logger.error(err) + except Exception as exc: # pylint: disable=broad-except logger.debug(traceback.format_exc()) + logger.warning( + f"Unexpected error filtering entities for table [{table}]: {exc}" + ) + self.source_status.filter(table.fullyQualifiedName.__root__, f"{exc}") def get_database_entities(self): """List all databases in service""" @@ -390,12 +395,16 @@ class ProfilerWorkflow: if hasattr(self, "sink"): self.sink.write_record(profile) self.status.processed(entity.fullyQualifiedName.__root__) - except Exception as err: # pylint: disable=broad-except - logger.error(err) - logger.error(traceback.format_exc()) - except Exception as err: # pylint: disable=broad-except - logger.error(err) + except Exception as exc: # pylint: disable=broad-except + logger.debug(traceback.format_exc()) + logger.warning( + f"Unexpected exception processing entity [{entity}]: {exc}" + ) + except Exception as exc: # pylint: disable=broad-except logger.debug(traceback.format_exc()) + logger.error( + f"Unexpected exception executing in database [{database}]: {exc}" + ) def print_status(self) -> int: """ diff --git a/ingestion/src/metadata/orm_profiler/metrics/static/count_in_set.py b/ingestion/src/metadata/orm_profiler/metrics/static/count_in_set.py index e9f6167ae82..d40619b1bdf 100644 --- a/ingestion/src/metadata/orm_profiler/metrics/static/count_in_set.py +++ b/ingestion/src/metadata/orm_profiler/metrics/static/count_in_set.py @@ -13,7 +13,7 @@ CountInSet Metric definition """ # pylint: disable=duplicate-code - +import traceback from typing import List from sqlalchemy import case, column, func @@ -57,6 +57,7 @@ class CountInSet(StaticMetric): set_values = set(self.values) return SumFn(case([(column(self.col.name).in_(set_values), 1)], else_=0)) - except Exception as err: # pylint: disable=broad-except - logger.error(f"Error trying to run countInSet for {self.col.name} - {err}") + except Exception as exc: # pylint: disable=broad-except + logger.debug(traceback.format_exc()) + logger.warning(f"Error trying to run countInSet for {self.col.name}: {exc}") return None diff --git a/ingestion/src/metadata/orm_profiler/orm/types/uuid.py b/ingestion/src/metadata/orm_profiler/orm/types/uuid.py index 0af8f15d98a..99a15c51f9a 100644 --- a/ingestion/src/metadata/orm_profiler/orm/types/uuid.py +++ b/ingestion/src/metadata/orm_profiler/orm/types/uuid.py @@ -13,7 +13,7 @@ Expand sqlalchemy types to map them to OpenMetadata DataType """ # pylint: disable=duplicate-code - +import traceback from uuid import UUID from sqlalchemy.sql.sqltypes import String, TypeDecorator @@ -43,7 +43,8 @@ class UUIDString(TypeDecorator): try: return UUID(value) except ValueError as err: - logger.error(f"Error converting value {value} to UUID") + logger.debug(traceback.format_exc()) + logger.error(f"Error converting value [{value}] to UUID: {err}") raise err def process_result_value(self, value: str, dialect): diff --git a/ingestion/src/metadata/orm_profiler/profiler/core.py b/ingestion/src/metadata/orm_profiler/profiler/core.py index 7826d875c17..9dbe722e427 100644 --- a/ingestion/src/metadata/orm_profiler/profiler/core.py +++ b/ingestion/src/metadata/orm_profiler/profiler/core.py @@ -12,6 +12,7 @@ """ Main Profile definition and queries to execute """ +import traceback import warnings from datetime import datetime, timezone from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Type @@ -433,7 +434,8 @@ class Profiler(Generic[TMetric]): ) except ValidationError as err: - logger.error(f"Cannot transform profiler results to TableProfile {err}") + logger.debug(traceback.format_exc()) + logger.error(f"Cannot transform profiler results to TableProfile: {err}") raise err @property diff --git a/ingestion/src/metadata/orm_profiler/sink/metadata_rest.py b/ingestion/src/metadata/orm_profiler/sink/metadata_rest.py index 3bc49cb875d..b336e18c95a 100644 --- a/ingestion/src/metadata/orm_profiler/sink/metadata_rest.py +++ b/ingestion/src/metadata/orm_profiler/sink/metadata_rest.py @@ -86,8 +86,8 @@ class MetadataRestSink(Sink[Entity]): ) except APIError as err: - logger.error( - f"Failed to sink profiler & test data for {record.table.fullyQualifiedName.__root__} - {err}" - ) logger.debug(traceback.format_exc()) + logger.warning( + f"Failed to sink profiler & test data for {record.table.fullyQualifiedName.__root__}: {err}" + ) self.status.failure(f"Table: {record.table.fullyQualifiedName.__root__}") diff --git a/ingestion/src/metadata/test_suite/api/workflow.py b/ingestion/src/metadata/test_suite/api/workflow.py index 70db97dae37..d9cc22bcfca 100644 --- a/ingestion/src/metadata/test_suite/api/workflow.py +++ b/ingestion/src/metadata/test_suite/api/workflow.py @@ -317,10 +317,10 @@ class TestSuiteWorkflow: ) ) except Exception as err: - logger.error( - f"Couldn't create test case name {test_case_name_to_create}" + logger.warning( + f"Couldn't create test case name {test_case_name_to_create}: {err}" ) - logger.error(traceback.format_exc(err)) + logger.debug(traceback.format_exc(err)) return created_test_case @@ -352,9 +352,8 @@ class TestSuiteWorkflow: logger.info(f"Successfuly ran test case {test_case.name.__root__}") self.status.processed(test_case.fullyQualifiedName.__root__) except Exception as err: - logger.error(traceback.format_exc(err)) - logger.error(f"Could not run test case {test_case.name}") - logger.error(err) + logger.debug(traceback.format_exc(err)) + logger.warning(f"Could not run test case {test_case.name}: {err}") self.status.failure(test_case.fullyQualifiedName.__root__) def print_status(self) -> int: diff --git a/ingestion/src/metadata/test_suite/validations/column/column_value_max_to_be_between.py b/ingestion/src/metadata/test_suite/validations/column/column_value_max_to_be_between.py index 24ee49cfcff..b03f18f3aae 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_value_max_to_be_between.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_value_max_to_be_between.py @@ -14,10 +14,10 @@ ColumnValuesToBeBetween validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import inspect -from sqlalchemy.orm import DeclarativeMeta from metadata.generated.schema.tests.basic import ( TestCaseResult, @@ -63,9 +63,10 @@ def column_value_max_to_be_between( except Exception as err: # pylint: disable=broad-except msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_value_mean_to_be_between.py b/ingestion/src/metadata/test_suite/validations/column/column_value_mean_to_be_between.py index d6170fea826..1e327ea298e 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_value_mean_to_be_between.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_value_mean_to_be_between.py @@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import inspect @@ -62,9 +63,10 @@ def column_value_mean_to_be_between( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_value_median_to_be_between.py b/ingestion/src/metadata/test_suite/validations/column/column_value_median_to_be_between.py index ec487845148..7f1e7e32d98 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_value_median_to_be_between.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_value_median_to_be_between.py @@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import inspect @@ -62,9 +63,10 @@ def column_value_median_to_be_between( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_value_min_to_be_between.py b/ingestion/src/metadata/test_suite/validations/column/column_value_min_to_be_between.py index 213e845b8de..aaf563073b0 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_value_min_to_be_between.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_value_min_to_be_between.py @@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import inspect @@ -62,9 +63,10 @@ def column_value_min_to_be_between( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_value_stddev_to_be_between.py b/ingestion/src/metadata/test_suite/validations/column/column_value_stddev_to_be_between.py index 8eb37fb2bb8..82aefe44b96 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_value_stddev_to_be_between.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_value_stddev_to_be_between.py @@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import inspect @@ -23,9 +24,6 @@ from metadata.generated.schema.tests.basic import ( TestCaseStatus, TestResultValue, ) -from metadata.generated.schema.tests.column.columnValueStdDevToBeBetween import ( - ColumnValueStdDevToBeBetween, -) from metadata.generated.schema.tests.testCase import TestCase from metadata.orm_profiler.metrics.registry import Metrics from metadata.orm_profiler.profiler.runner import QueryRunner @@ -65,9 +63,10 @@ def column_value_stddev_to_be_between( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_values_in_set.py b/ingestion/src/metadata/test_suite/validations/column/column_values_in_set.py index b3f23c5ee9c..d1feaf9d80e 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_values_in_set.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_values_in_set.py @@ -13,7 +13,7 @@ ColumnValuesToBeNotNull validation implementation """ # pylint: disable=duplicate-code - +import traceback from ast import literal_eval from datetime import datetime @@ -24,9 +24,6 @@ from metadata.generated.schema.tests.basic import ( TestCaseStatus, TestResultValue, ) -from metadata.generated.schema.tests.column.columnValuesToBeInSet import ( - ColumnValuesToBeInSet, -) from metadata.generated.schema.tests.testCase import TestCase from metadata.orm_profiler.metrics.core import add_props from metadata.orm_profiler.metrics.registry import Metrics @@ -77,9 +74,10 @@ def column_values_in_set( except Exception as err: # pylint: disable=broad-except msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_values_length_to_be_between.py b/ingestion/src/metadata/test_suite/validations/column/column_values_length_to_be_between.py index 3cd3bc44662..8662e69cfbf 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_values_length_to_be_between.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_values_length_to_be_between.py @@ -14,6 +14,7 @@ ColumnValueLengthsToBeBetween validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import inspect @@ -70,9 +71,10 @@ def column_value_length_to_be_between( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_values_missing_count_to_be_equal.py b/ingestion/src/metadata/test_suite/validations/column/column_values_missing_count_to_be_equal.py index 285732d3f75..58d84921de9 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_values_missing_count_to_be_equal.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_values_missing_count_to_be_equal.py @@ -12,9 +12,9 @@ """ ColumnValuesMissingCount validation implementation """ -from ast import literal_eval - # pylint: disable=duplicate-code +import traceback +from ast import literal_eval from datetime import datetime from sqlalchemy import inspect @@ -24,9 +24,6 @@ from metadata.generated.schema.tests.basic import ( TestCaseStatus, TestResultValue, ) -from metadata.generated.schema.tests.column.columnValuesMissingCountToBeEqual import ( - ColumnValuesMissingCount, -) from metadata.generated.schema.tests.testCase import TestCase from metadata.orm_profiler.metrics.core import add_props from metadata.orm_profiler.metrics.registry import Metrics @@ -69,9 +66,10 @@ def column_values_missing_count_to_be_equal( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, @@ -108,8 +106,9 @@ def column_values_missing_count_to_be_equal( null_count_value_res += set_count_res except Exception as err: # pylint: disable=broad-except - msg = f"Error computing {test_case.__class__.__name__} for {runner.table.__tablename__} - {err}" - logger.error(msg) + msg = f"Error computing {test_case.__class__.__name__} for {runner.table.__tablename__}: {err}" + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_values_not_in_set.py b/ingestion/src/metadata/test_suite/validations/column/column_values_not_in_set.py index d7e02721729..0e4ed53d815 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_values_not_in_set.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_values_not_in_set.py @@ -13,7 +13,7 @@ ColumnValuesToBeNotNull validation implementation """ # pylint: disable=duplicate-code - +import traceback from ast import literal_eval from datetime import datetime @@ -73,9 +73,10 @@ def column_values_not_in_set( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_values_sum_to_be_between.py b/ingestion/src/metadata/test_suite/validations/column/column_values_sum_to_be_between.py index df99030d0f6..895e362bdbe 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_values_sum_to_be_between.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_values_sum_to_be_between.py @@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import inspect @@ -62,9 +63,10 @@ def column_values_sum_to_be_between( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_values_to_be_between.py b/ingestion/src/metadata/test_suite/validations/column/column_values_to_be_between.py index aca5d0c4514..8729c8be5ac 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_values_to_be_between.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_values_to_be_between.py @@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import inspect @@ -23,9 +24,6 @@ from metadata.generated.schema.tests.basic import ( TestCaseStatus, TestResultValue, ) -from metadata.generated.schema.tests.column.columnValuesToBeBetween import ( - ColumnValuesToBeBetween, -) from metadata.generated.schema.tests.testCase import TestCase from metadata.orm_profiler.metrics.registry import Metrics from metadata.orm_profiler.profiler.runner import QueryRunner @@ -69,9 +67,10 @@ def column_values_to_be_between( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_values_to_be_not_null.py b/ingestion/src/metadata/test_suite/validations/column/column_values_to_be_not_null.py index 4502b6c308b..945d7b30641 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_values_to_be_not_null.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_values_to_be_not_null.py @@ -14,6 +14,7 @@ ColumnValuesToBeNotNull validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import inspect @@ -23,9 +24,6 @@ from metadata.generated.schema.tests.basic import ( TestCaseStatus, TestResultValue, ) -from metadata.generated.schema.tests.column.columnValuesToBeNotNull import ( - ColumnValuesToBeNotNull, -) from metadata.generated.schema.tests.testCase import TestCase from metadata.orm_profiler.metrics.registry import Metrics from metadata.orm_profiler.profiler.runner import QueryRunner @@ -65,9 +63,10 @@ def column_values_to_be_not_null( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_values_to_be_unique.py b/ingestion/src/metadata/test_suite/validations/column/column_values_to_be_unique.py index 617907f1649..94cfd555148 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_values_to_be_unique.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_values_to_be_unique.py @@ -14,6 +14,7 @@ ColumnValuesToBeUnique validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import inspect @@ -24,9 +25,6 @@ from metadata.generated.schema.tests.basic import ( TestCaseStatus, TestResultValue, ) -from metadata.generated.schema.tests.column.columnValuesToBeUnique import ( - ColumnValuesToBeUnique, -) from metadata.generated.schema.tests.testCase import TestCase from metadata.orm_profiler.metrics.registry import Metrics from metadata.orm_profiler.profiler.runner import QueryRunner @@ -76,8 +74,9 @@ def column_values_to_be_unique( unique_count_value_res = unique_count_value_dict.get(Metrics.UNIQUE_COUNT.name) except Exception as err: - msg = f"Error computing {test_case.name.__root__} for {runner.table.__tablename__} - {err}" - logger.error(msg) + msg = f"Error computing {test_case.name.__root__} for {runner.table.__tablename__}: {err}" + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_values_to_match_regex.py b/ingestion/src/metadata/test_suite/validations/column/column_values_to_match_regex.py index a3525ecc565..51fd389b651 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_values_to_match_regex.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_values_to_match_regex.py @@ -13,7 +13,7 @@ ColumnValuesToBeNotNull validation implementation """ # pylint: disable=duplicate-code - +import traceback from datetime import datetime from sqlalchemy import inspect @@ -23,9 +23,6 @@ from metadata.generated.schema.tests.basic import ( TestCaseStatus, TestResultValue, ) -from metadata.generated.schema.tests.column.columnValuesToMatchRegex import ( - ColumnValuesToMatchRegex, -) from metadata.generated.schema.tests.testCase import TestCase from metadata.orm_profiler.metrics.core import add_props from metadata.orm_profiler.metrics.registry import Metrics @@ -78,9 +75,10 @@ def column_values_to_match_regex( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/column/column_values_to_not_match_regex.py b/ingestion/src/metadata/test_suite/validations/column/column_values_to_not_match_regex.py index 5b9cbcb60cc..523aaf8ac6e 100644 --- a/ingestion/src/metadata/test_suite/validations/column/column_values_to_not_match_regex.py +++ b/ingestion/src/metadata/test_suite/validations/column/column_values_to_not_match_regex.py @@ -13,7 +13,7 @@ ColumnValuesToBeNotNull validation implementation """ # pylint: disable=duplicate-code - +import traceback from datetime import datetime from sqlalchemy import inspect @@ -75,9 +75,10 @@ def column_values_to_not_match_regex( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/table/table_column_count_to_be_between.py b/ingestion/src/metadata/test_suite/validations/table/table_column_count_to_be_between.py index 15fbdde7c68..4ffafec036b 100644 --- a/ingestion/src/metadata/test_suite/validations/table/table_column_count_to_be_between.py +++ b/ingestion/src/metadata/test_suite/validations/table/table_column_count_to_be_between.py @@ -14,6 +14,7 @@ TableColumnCountToBeBetween validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import inspect @@ -51,9 +52,10 @@ def table_column_count_to_be_between( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/table/table_column_count_to_equal.py b/ingestion/src/metadata/test_suite/validations/table/table_column_count_to_equal.py index f82d1425a30..97888118a10 100644 --- a/ingestion/src/metadata/test_suite/validations/table/table_column_count_to_equal.py +++ b/ingestion/src/metadata/test_suite/validations/table/table_column_count_to_equal.py @@ -14,6 +14,7 @@ TableColumnCountToEqual validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import inspect @@ -48,9 +49,10 @@ def table_column_count_to_equal( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/table/table_column_name_to_exist.py b/ingestion/src/metadata/test_suite/validations/table/table_column_name_to_exist.py index 603df35ec2a..13fa4a908e1 100644 --- a/ingestion/src/metadata/test_suite/validations/table/table_column_name_to_exist.py +++ b/ingestion/src/metadata/test_suite/validations/table/table_column_name_to_exist.py @@ -15,6 +15,7 @@ TableColumnCountToBeBetween validation implementation # pylint: disable=duplicate-code import reprlib +import traceback from datetime import datetime from typing import List @@ -65,9 +66,10 @@ def table_column_name_to_exist( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/table/table_column_to_match_set.py b/ingestion/src/metadata/test_suite/validations/table/table_column_to_match_set.py index 25b194adfbd..411397e7b07 100644 --- a/ingestion/src/metadata/test_suite/validations/table/table_column_to_match_set.py +++ b/ingestion/src/metadata/test_suite/validations/table/table_column_to_match_set.py @@ -16,6 +16,7 @@ TableColumnCountToBeBetween validation implementation import collections import reprlib +import traceback from datetime import datetime from typing import List @@ -66,9 +67,10 @@ def table_column_to_match_set( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/table/table_custom_sql_query.py b/ingestion/src/metadata/test_suite/validations/table/table_custom_sql_query.py index b0e5e7ad61c..47357625bd6 100644 --- a/ingestion/src/metadata/test_suite/validations/table/table_custom_sql_query.py +++ b/ingestion/src/metadata/test_suite/validations/table/table_custom_sql_query.py @@ -14,6 +14,7 @@ TableColumnCountToBeBetween validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from sqlalchemy import text @@ -60,9 +61,10 @@ def table_custom_sql_query( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/table/table_row_count_to_be_between.py b/ingestion/src/metadata/test_suite/validations/table/table_row_count_to_be_between.py index ff8be516cc7..19dd2b70b68 100644 --- a/ingestion/src/metadata/test_suite/validations/table/table_row_count_to_be_between.py +++ b/ingestion/src/metadata/test_suite/validations/table/table_row_count_to_be_between.py @@ -14,6 +14,7 @@ TableRowCountToBeBetween validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from metadata.generated.schema.tests.basic import ( @@ -50,9 +51,10 @@ def table_row_count_to_be_between( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/test_suite/validations/table/table_row_count_to_equal.py b/ingestion/src/metadata/test_suite/validations/table/table_row_count_to_equal.py index 8ce23f3199e..ea06fdbdcab 100644 --- a/ingestion/src/metadata/test_suite/validations/table/table_row_count_to_equal.py +++ b/ingestion/src/metadata/test_suite/validations/table/table_row_count_to_equal.py @@ -14,6 +14,7 @@ TableRowCountToEqual validation implementation """ # pylint: disable=duplicate-code +import traceback from datetime import datetime from metadata.generated.schema.tests.basic import ( @@ -50,9 +51,10 @@ def table_row_count_to_equal( except Exception as err: msg = ( - f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}" + f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}" ) - logger.error(msg) + logger.debug(traceback.format_exc()) + logger.warning(msg) return TestCaseResult( timestamp=execution_date, testCaseStatus=TestCaseStatus.Aborted,