Fix#6027: Improve logging in ORM profiler (#6919)

* Improve logging in ORM profiler

* Fix failing test

* Updated logging in test_suite module to match company's format

Co-authored-by: Teddy Crepineau <teddy.crepineau@gmail.com>

ingestion/src/metadata/interfaces/sqa_interface.py

@@ -16,6 +16,7 @@ supporting sqlalchemy abstraction layer
 
 import concurrent.futures
 import threading
+import traceback
 from collections import defaultdict
 from datetime import datetime, timezone
 from typing import Dict, List, Optional, Union
@@ -164,9 +165,10 @@ class SQAInterface(InterfaceProtocol):
         try:
             self._metadata = OpenMetadata(metadata_config)
             self._metadata.health_check()
-        except Exception:
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
             logger.warning(
-                "No OpenMetadata server configuration found. Running profiler interface without OM server connection"
+                f"No OpenMetadata server configuration found. Running profiler interface without OM server connection: {exc}"
             )
             self._metadata = None
 
@@ -248,8 +250,9 @@ class SQAInterface(InterfaceProtocol):
 
         try:
             column = column.name
-        except Exception as err:
+        except Exception as exc:
-            logger.debug(err)
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Unexpected exception computing metrics: {exc}")
 
         return row, column
 
@@ -331,8 +334,9 @@ class SQAInterface(InterfaceProtocol):
         """
         try:
             return metric(column).fn(column_results)
-        except Exception as err:
+        except Exception as exc:
-            logger.error(err)
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Unexpected exception computing metrics: {exc}")
             self.session.rollback()
 
     def run_test_case(
@@ -382,9 +386,10 @@ def get_table_metrics(
         if row:
             return dict(row)
 
-    except Exception as err:
+    except Exception as exc:
-        logger.error(
+        logger.debug(traceback.format_exc())
-            f"Error trying to compute profile for {runner.table.__tablename__} - {err}"
+        logger.warning(
+            f"Error trying to compute profile for {runner.table.__tablename__}: {exc}"
         )
         session.rollback()
 
@@ -415,9 +420,10 @@ def get_static_metrics(
             ]
         )
         return dict(row)
-    except Exception as err:
+    except Exception as exc:
-        logger.error(
+        logger.debug(traceback.format_exc())
-            f"Error trying to compute profile for {runner.table.__tablename__}.{column.name} - {err}"
+        logger.warning(
+            f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
         )
         session.rollback()
 
@@ -453,9 +459,10 @@ def get_query_metrics(
         else:
             row = runner.select_first_from_query(metric_query)
             return dict(row)
-    except Exception as err:
+    except Exception as exc:
-        logger.error(
+        logger.debug(traceback.format_exc())
-            f"Error trying to compute profile for {runner.table.__tablename__}.{column.name} - {err}"
+        logger.warning(
+            f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
         )
         session.rollback()
 
@@ -482,9 +489,10 @@ def get_window_metrics(
         if not isinstance(row, Row):
             return {metric.name(): row}
         return dict(row)
-    except Exception as err:
+    except Exception as exc:
-        logger.error(
+        logger.debug(traceback.format_exc())
-            f"Error trying to compute profile for {runner.table.__tablename__}.{column.name} - {err}"
+        logger.warning(
+            f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
         )
         session.rollback()
 

ingestion/src/metadata/orm_profiler/api/workflow.py

@@ -120,7 +120,10 @@ class ProfilerWorkflow:
             config = parse_workflow_config_gracefully(config_dict)
             return cls(config)
         except ValidationError as err:
-            logger.error("Error trying to parse the Profiler Workflow configuration")
+            logger.debug(traceback.format_exc())
+            logger.error(
+                f"Error trying to parse the Profiler Workflow configuration: {err}"
+            )
             raise err
 
     def get_config_for_entity(self, entity: Table) -> Optional[TableConfig]:
@@ -293,10 +296,12 @@ class ProfilerWorkflow:
 
                 self.source_status.scanned(table.fullyQualifiedName.__root__)
                 yield table
-            except Exception as err:  # pylint: disable=broad-except
+            except Exception as exc:  # pylint: disable=broad-except
-                self.source_status.filter(table.fullyQualifiedName.__root__, f"{err}")
-                logger.error(err)
                 logger.debug(traceback.format_exc())
+                logger.warning(
+                    f"Unexpected error filtering entities for table [{table}]: {exc}"
+                )
+                self.source_status.filter(table.fullyQualifiedName.__root__, f"{exc}")
 
     def get_database_entities(self):
         """List all databases in service"""
@@ -390,12 +395,16 @@ class ProfilerWorkflow:
                         if hasattr(self, "sink"):
                             self.sink.write_record(profile)
                         self.status.processed(entity.fullyQualifiedName.__root__)
-                    except Exception as err:  # pylint: disable=broad-except
+                    except Exception as exc:  # pylint: disable=broad-except
-                        logger.error(err)
-                        logger.error(traceback.format_exc())
-            except Exception as err:  # pylint: disable=broad-except
-                logger.error(err)
                         logger.debug(traceback.format_exc())
+                        logger.warning(
+                            f"Unexpected exception processing entity [{entity}]: {exc}"
+                        )
+            except Exception as exc:  # pylint: disable=broad-except
+                logger.debug(traceback.format_exc())
+                logger.error(
+                    f"Unexpected exception executing in database [{database}]: {exc}"
+                )
 
     def print_status(self) -> int:
         """

ingestion/src/metadata/orm_profiler/metrics/static/count_in_set.py

@@ -13,7 +13,7 @@
 CountInSet Metric definition
 """
 # pylint: disable=duplicate-code
-
+import traceback
 from typing import List
 
 from sqlalchemy import case, column, func
@@ -57,6 +57,7 @@ class CountInSet(StaticMetric):
             set_values = set(self.values)
             return SumFn(case([(column(self.col.name).in_(set_values), 1)], else_=0))
 
-        except Exception as err:  # pylint: disable=broad-except
+        except Exception as exc:  # pylint: disable=broad-except
-            logger.error(f"Error trying to run countInSet for {self.col.name} - {err}")
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Error trying to run countInSet for {self.col.name}: {exc}")
             return None

ingestion/src/metadata/orm_profiler/orm/types/uuid.py

@@ -13,7 +13,7 @@
 Expand sqlalchemy types to map them to OpenMetadata DataType
 """
 # pylint: disable=duplicate-code
-
+import traceback
 from uuid import UUID
 
 from sqlalchemy.sql.sqltypes import String, TypeDecorator
@@ -43,7 +43,8 @@ class UUIDString(TypeDecorator):
         try:
             return UUID(value)
         except ValueError as err:
-            logger.error(f"Error converting value {value} to UUID")
+            logger.debug(traceback.format_exc())
+            logger.error(f"Error converting value [{value}] to UUID: {err}")
             raise err
 
     def process_result_value(self, value: str, dialect):

ingestion/src/metadata/orm_profiler/profiler/core.py

@@ -12,6 +12,7 @@
 """
 Main Profile definition and queries to execute
 """
+import traceback
 import warnings
 from datetime import datetime, timezone
 from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Type
@@ -433,7 +434,8 @@ class Profiler(Generic[TMetric]):
             )
 
         except ValidationError as err:
-            logger.error(f"Cannot transform profiler results to TableProfile {err}")
+            logger.debug(traceback.format_exc())
+            logger.error(f"Cannot transform profiler results to TableProfile: {err}")
             raise err
 
     @property

ingestion/src/metadata/orm_profiler/sink/metadata_rest.py

@@ -86,8 +86,8 @@ class MetadataRestSink(Sink[Entity]):
             )
 
         except APIError as err:
-            logger.error(
-                f"Failed to sink profiler & test data for {record.table.fullyQualifiedName.__root__} - {err}"
-            )
             logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Failed to sink profiler & test data for {record.table.fullyQualifiedName.__root__}: {err}"
+            )
             self.status.failure(f"Table: {record.table.fullyQualifiedName.__root__}")

ingestion/src/metadata/test_suite/api/workflow.py

@@ -317,10 +317,10 @@ class TestSuiteWorkflow:
                     )
                 )
             except Exception as err:
-                logger.error(
+                logger.warning(
-                    f"Couldn't create test case name {test_case_name_to_create}"
+                    f"Couldn't create test case name {test_case_name_to_create}: {err}"
                 )
-                logger.error(traceback.format_exc(err))
+                logger.debug(traceback.format_exc(err))
 
         return created_test_case
 
@@ -352,9 +352,8 @@ class TestSuiteWorkflow:
                     logger.info(f"Successfuly ran test case {test_case.name.__root__}")
                     self.status.processed(test_case.fullyQualifiedName.__root__)
                 except Exception as err:
-                    logger.error(traceback.format_exc(err))
+                    logger.debug(traceback.format_exc(err))
-                    logger.error(f"Could not run test case {test_case.name}")
+                    logger.warning(f"Could not run test case {test_case.name}: {err}")
-                    logger.error(err)
                     self.status.failure(test_case.fullyQualifiedName.__root__)
 
     def print_status(self) -> int:

ingestion/src/metadata/test_suite/validations/column/column_value_max_to_be_between.py

@@ -14,10 +14,10 @@ ColumnValuesToBeBetween validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
-from sqlalchemy.orm import DeclarativeMeta
 
 from metadata.generated.schema.tests.basic import (
     TestCaseResult,
@@ -63,9 +63,10 @@ def column_value_max_to_be_between(
 
     except Exception as err:  # pylint: disable=broad-except
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_value_mean_to_be_between.py

@@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -62,9 +63,10 @@ def column_value_mean_to_be_between(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_value_median_to_be_between.py

@@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -62,9 +63,10 @@ def column_value_median_to_be_between(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_value_min_to_be_between.py

@@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -62,9 +63,10 @@ def column_value_min_to_be_between(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_value_stddev_to_be_between.py

@@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -23,9 +24,6 @@ from metadata.generated.schema.tests.basic import (
     TestCaseStatus,
     TestResultValue,
 )
-from metadata.generated.schema.tests.column.columnValueStdDevToBeBetween import (
-    ColumnValueStdDevToBeBetween,
-)
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.orm_profiler.metrics.registry import Metrics
 from metadata.orm_profiler.profiler.runner import QueryRunner
@@ -65,9 +63,10 @@ def column_value_stddev_to_be_between(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_values_in_set.py

@@ -13,7 +13,7 @@
 ColumnValuesToBeNotNull validation implementation
 """
 # pylint: disable=duplicate-code
-
+import traceback
 from ast import literal_eval
 from datetime import datetime
 
@@ -24,9 +24,6 @@ from metadata.generated.schema.tests.basic import (
     TestCaseStatus,
     TestResultValue,
 )
-from metadata.generated.schema.tests.column.columnValuesToBeInSet import (
-    ColumnValuesToBeInSet,
-)
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.orm_profiler.metrics.core import add_props
 from metadata.orm_profiler.metrics.registry import Metrics
@@ -77,9 +74,10 @@ def column_values_in_set(
 
     except Exception as err:  # pylint: disable=broad-except
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_values_length_to_be_between.py

@@ -14,6 +14,7 @@ ColumnValueLengthsToBeBetween validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -70,9 +71,10 @@ def column_value_length_to_be_between(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_values_missing_count_to_be_equal.py

@@ -12,9 +12,9 @@
 """
 ColumnValuesMissingCount validation implementation
 """
-from ast import literal_eval
-
 # pylint: disable=duplicate-code
+import traceback
+from ast import literal_eval
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -24,9 +24,6 @@ from metadata.generated.schema.tests.basic import (
     TestCaseStatus,
     TestResultValue,
 )
-from metadata.generated.schema.tests.column.columnValuesMissingCountToBeEqual import (
-    ColumnValuesMissingCount,
-)
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.orm_profiler.metrics.core import add_props
 from metadata.orm_profiler.metrics.registry import Metrics
@@ -69,9 +66,10 @@ def column_values_missing_count_to_be_equal(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,
@@ -108,8 +106,9 @@ def column_values_missing_count_to_be_equal(
             null_count_value_res += set_count_res
 
         except Exception as err:  # pylint: disable=broad-except
-            msg = f"Error computing {test_case.__class__.__name__} for {runner.table.__tablename__} - {err}"
+            msg = f"Error computing {test_case.__class__.__name__} for {runner.table.__tablename__}: {err}"
-            logger.error(msg)
+            logger.debug(traceback.format_exc())
+            logger.warning(msg)
             return TestCaseResult(
                 timestamp=execution_date,
                 testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_values_not_in_set.py

@@ -13,7 +13,7 @@
 ColumnValuesToBeNotNull validation implementation
 """
 # pylint: disable=duplicate-code
-
+import traceback
 from ast import literal_eval
 from datetime import datetime
 
@@ -73,9 +73,10 @@ def column_values_not_in_set(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_values_sum_to_be_between.py

@@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -62,9 +63,10 @@ def column_values_sum_to_be_between(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_values_to_be_between.py

@@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -23,9 +24,6 @@ from metadata.generated.schema.tests.basic import (
     TestCaseStatus,
     TestResultValue,
 )
-from metadata.generated.schema.tests.column.columnValuesToBeBetween import (
-    ColumnValuesToBeBetween,
-)
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.orm_profiler.metrics.registry import Metrics
 from metadata.orm_profiler.profiler.runner import QueryRunner
@@ -69,9 +67,10 @@ def column_values_to_be_between(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_values_to_be_not_null.py

@@ -14,6 +14,7 @@ ColumnValuesToBeNotNull validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -23,9 +24,6 @@ from metadata.generated.schema.tests.basic import (
     TestCaseStatus,
     TestResultValue,
 )
-from metadata.generated.schema.tests.column.columnValuesToBeNotNull import (
-    ColumnValuesToBeNotNull,
-)
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.orm_profiler.metrics.registry import Metrics
 from metadata.orm_profiler.profiler.runner import QueryRunner
@@ -65,9 +63,10 @@ def column_values_to_be_not_null(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_values_to_be_unique.py

@@ -14,6 +14,7 @@ ColumnValuesToBeUnique validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -24,9 +25,6 @@ from metadata.generated.schema.tests.basic import (
     TestCaseStatus,
     TestResultValue,
 )
-from metadata.generated.schema.tests.column.columnValuesToBeUnique import (
-    ColumnValuesToBeUnique,
-)
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.orm_profiler.metrics.registry import Metrics
 from metadata.orm_profiler.profiler.runner import QueryRunner
@@ -76,8 +74,9 @@ def column_values_to_be_unique(
         unique_count_value_res = unique_count_value_dict.get(Metrics.UNIQUE_COUNT.name)
 
     except Exception as err:
-        msg = f"Error computing {test_case.name.__root__} for {runner.table.__tablename__} - {err}"
+        msg = f"Error computing {test_case.name.__root__} for {runner.table.__tablename__}: {err}"
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_values_to_match_regex.py

@@ -13,7 +13,7 @@
 ColumnValuesToBeNotNull validation implementation
 """
 # pylint: disable=duplicate-code
-
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -23,9 +23,6 @@ from metadata.generated.schema.tests.basic import (
     TestCaseStatus,
     TestResultValue,
 )
-from metadata.generated.schema.tests.column.columnValuesToMatchRegex import (
-    ColumnValuesToMatchRegex,
-)
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.orm_profiler.metrics.core import add_props
 from metadata.orm_profiler.metrics.registry import Metrics
@@ -78,9 +75,10 @@ def column_values_to_match_regex(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/column/column_values_to_not_match_regex.py

@@ -13,7 +13,7 @@
 ColumnValuesToBeNotNull validation implementation
 """
 # pylint: disable=duplicate-code
-
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -75,9 +75,10 @@ def column_values_to_not_match_regex(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/table/table_column_count_to_be_between.py

@@ -14,6 +14,7 @@ TableColumnCountToBeBetween validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -51,9 +52,10 @@ def table_column_count_to_be_between(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/table/table_column_count_to_equal.py

@@ -14,6 +14,7 @@ TableColumnCountToEqual validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import inspect
@@ -48,9 +49,10 @@ def table_column_count_to_equal(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/table/table_column_name_to_exist.py

@@ -15,6 +15,7 @@ TableColumnCountToBeBetween validation implementation
 # pylint: disable=duplicate-code
 
 import reprlib
+import traceback
 from datetime import datetime
 from typing import List
 
@@ -65,9 +66,10 @@ def table_column_name_to_exist(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/table/table_column_to_match_set.py

@@ -16,6 +16,7 @@ TableColumnCountToBeBetween validation implementation
 
 import collections
 import reprlib
+import traceback
 from datetime import datetime
 from typing import List
 
@@ -66,9 +67,10 @@ def table_column_to_match_set(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/table/table_custom_sql_query.py

@@ -14,6 +14,7 @@ TableColumnCountToBeBetween validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from sqlalchemy import text
@@ -60,9 +61,10 @@ def table_custom_sql_query(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/table/table_row_count_to_be_between.py

@@ -14,6 +14,7 @@ TableRowCountToBeBetween validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from metadata.generated.schema.tests.basic import (
@@ -50,9 +51,10 @@ def table_row_count_to_be_between(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,

ingestion/src/metadata/test_suite/validations/table/table_row_count_to_equal.py

@@ -14,6 +14,7 @@ TableRowCountToEqual validation implementation
 """
 # pylint: disable=duplicate-code
 
+import traceback
 from datetime import datetime
 
 from metadata.generated.schema.tests.basic import (
@@ -50,9 +51,10 @@ def table_row_count_to_equal(
 
     except Exception as err:
         msg = (
-            f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
+            f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
         )
-        logger.error(msg)
+        logger.debug(traceback.format_exc())
+        logger.warning(msg)
         return TestCaseResult(
             timestamp=execution_date,
             testCaseStatus=TestCaseStatus.Aborted,