yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-26 09:22:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix#6027: Improve logging in ORM profiler (#6919)

Browse Source 
* Improve logging in ORM profiler

* Fix failing test

* Updated logging in test_suite module to match company's format

Co-authored-by: Teddy Crepineau <teddy.crepineau@gmail.com>
This commit is contained in:
Nahuel 2022-08-25 11:19:31 +02:00 committed by GitHub
parent ce578e73d4
commit 7863f040d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 154 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
ingestion/src/metadata/interfaces/sqa_interface.py
View File

@ -16,6 +16,7 @@ supporting sqlalchemy abstraction layer
import concurrent.futures
import threading
import traceback
from collections import defaultdict
from datetime import datetime, timezone
from typing import Dict, List, Optional, Union
@ -164,9 +165,10 @@ class SQAInterface(InterfaceProtocol):
try:
self._metadata = OpenMetadata(metadata_config)
self._metadata.health_check()
except Exception:
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(
"No OpenMetadata server configuration found. Running profiler interface without OM server connection"
f"No OpenMetadata server configuration found. Running profiler interface without OM server connection: {exc}"
)
self._metadata = None
@ -248,8 +250,9 @@ class SQAInterface(InterfaceProtocol):
try:
column = column.name
except Exception as err:
logger.debug(err)
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(f"Unexpected exception computing metrics: {exc}")
return row, column
@ -331,8 +334,9 @@ class SQAInterface(InterfaceProtocol):
"""
try:
return metric(column).fn(column_results)
except Exception as err:
logger.error(err)
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(f"Unexpected exception computing metrics: {exc}")
self.session.rollback()
def run_test_case(
@ -382,9 +386,10 @@ def get_table_metrics(
if row:
return dict(row)
except Exception as err:
logger.error(
f"Error trying to compute profile for {runner.table.__tablename__} - {err}"
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(
f"Error trying to compute profile for {runner.table.__tablename__}: {exc}"
)
session.rollback()
@ -415,9 +420,10 @@ def get_static_metrics(
]
)
return dict(row)
except Exception as err:
logger.error(
f"Error trying to compute profile for {runner.table.__tablename__}.{column.name} - {err}"
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(
f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
)
session.rollback()
@ -453,9 +459,10 @@ def get_query_metrics(
else:
row = runner.select_first_from_query(metric_query)
return dict(row)
except Exception as err:
logger.error(
f"Error trying to compute profile for {runner.table.__tablename__}.{column.name} - {err}"
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(
f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
)
session.rollback()
@ -482,9 +489,10 @@ def get_window_metrics(
if not isinstance(row, Row):
return {metric.name(): row}
return dict(row)
except Exception as err:
logger.error(
f"Error trying to compute profile for {runner.table.__tablename__}.{column.name} - {err}"
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(
f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
)
session.rollback()

27
ingestion/src/metadata/orm_profiler/api/workflow.py
View File

@ -120,7 +120,10 @@ class ProfilerWorkflow:
config = parse_workflow_config_gracefully(config_dict)
return cls(config)
except ValidationError as err:
logger.error("Error trying to parse the Profiler Workflow configuration")
logger.debug(traceback.format_exc())
logger.error(
f"Error trying to parse the Profiler Workflow configuration: {err}"
)
raise err
def get_config_for_entity(self, entity: Table) -> Optional[TableConfig]:
@ -293,10 +296,12 @@ class ProfilerWorkflow:
self.source_status.scanned(table.fullyQualifiedName.__root__)
yield table
except Exception as err: # pylint: disable=broad-except
self.source_status.filter(table.fullyQualifiedName.__root__, f"{err}")
logger.error(err)
except Exception as exc: # pylint: disable=broad-except
logger.debug(traceback.format_exc())
logger.warning(
f"Unexpected error filtering entities for table [{table}]: {exc}"
)
self.source_status.filter(table.fullyQualifiedName.__root__, f"{exc}")
def get_database_entities(self):
"""List all databases in service"""
@ -390,12 +395,16 @@ class ProfilerWorkflow:
if hasattr(self, "sink"):
self.sink.write_record(profile)
self.status.processed(entity.fullyQualifiedName.__root__)
except Exception as err: # pylint: disable=broad-except
logger.error(err)
logger.error(traceback.format_exc())
except Exception as err: # pylint: disable=broad-except
logger.error(err)
except Exception as exc: # pylint: disable=broad-except
logger.debug(traceback.format_exc())
logger.warning(
f"Unexpected exception processing entity [{entity}]: {exc}"
)
except Exception as exc: # pylint: disable=broad-except
logger.debug(traceback.format_exc())
logger.error(
f"Unexpected exception executing in database [{database}]: {exc}"
)
def print_status(self) -> int:
"""

7
ingestion/src/metadata/orm_profiler/metrics/static/count_in_set.py
View File

@ -13,7 +13,7 @@
CountInSet Metric definition
"""
# pylint: disable=duplicate-code
import traceback
from typing import List
from sqlalchemy import case, column, func
@ -57,6 +57,7 @@ class CountInSet(StaticMetric):
set_values = set(self.values)
return SumFn(case([(column(self.col.name).in_(set_values), 1)], else_=0))
except Exception as err: # pylint: disable=broad-except
logger.error(f"Error trying to run countInSet for {self.col.name} - {err}")
except Exception as exc: # pylint: disable=broad-except
logger.debug(traceback.format_exc())
logger.warning(f"Error trying to run countInSet for {self.col.name}: {exc}")
return None

5
ingestion/src/metadata/orm_profiler/orm/types/uuid.py
View File

@ -13,7 +13,7 @@
Expand sqlalchemy types to map them to OpenMetadata DataType
"""
# pylint: disable=duplicate-code
import traceback
from uuid import UUID
from sqlalchemy.sql.sqltypes import String, TypeDecorator
@ -43,7 +43,8 @@ class UUIDString(TypeDecorator):
try:
return UUID(value)
except ValueError as err:
logger.error(f"Error converting value {value} to UUID")
logger.debug(traceback.format_exc())
logger.error(f"Error converting value [{value}] to UUID: {err}")
raise err
def process_result_value(self, value: str, dialect):

4
ingestion/src/metadata/orm_profiler/profiler/core.py
View File

@ -12,6 +12,7 @@
"""
Main Profile definition and queries to execute
"""
import traceback
import warnings
from datetime import datetime, timezone
from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Type
@ -433,7 +434,8 @@ class Profiler(Generic[TMetric]):
)
except ValidationError as err:
logger.error(f"Cannot transform profiler results to TableProfile {err}")
logger.debug(traceback.format_exc())
logger.error(f"Cannot transform profiler results to TableProfile: {err}")
raise err
@property

6
ingestion/src/metadata/orm_profiler/sink/metadata_rest.py
View File

@ -86,8 +86,8 @@ class MetadataRestSink(Sink[Entity]):
)
except APIError as err:
logger.error(
f"Failed to sink profiler & test data for {record.table.fullyQualifiedName.__root__} - {err}"
)
logger.debug(traceback.format_exc())
logger.warning(
f"Failed to sink profiler & test data for {record.table.fullyQualifiedName.__root__}: {err}"
)
self.status.failure(f"Table: {record.table.fullyQualifiedName.__root__}")

11
ingestion/src/metadata/test_suite/api/workflow.py
View File

@ -317,10 +317,10 @@ class TestSuiteWorkflow:
)
)
except Exception as err:
logger.error(
f"Couldn't create test case name {test_case_name_to_create}"
logger.warning(
f"Couldn't create test case name {test_case_name_to_create}: {err}"
)
logger.error(traceback.format_exc(err))
logger.debug(traceback.format_exc(err))
return created_test_case
@ -352,9 +352,8 @@ class TestSuiteWorkflow:
logger.info(f"Successfuly ran test case {test_case.name.__root__}")
self.status.processed(test_case.fullyQualifiedName.__root__)
except Exception as err:
logger.error(traceback.format_exc(err))
logger.error(f"Could not run test case {test_case.name}")
logger.error(err)
logger.debug(traceback.format_exc(err))
logger.warning(f"Could not run test case {test_case.name}: {err}")
self.status.failure(test_case.fullyQualifiedName.__root__)
def print_status(self) -> int:

7
ingestion/src/metadata/test_suite/validations/column/column_value_max_to_be_between.py
View File

@ -14,10 +14,10 @@ ColumnValuesToBeBetween validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
from sqlalchemy.orm import DeclarativeMeta
from metadata.generated.schema.tests.basic import (
TestCaseResult,
@ -63,9 +63,10 @@ def column_value_max_to_be_between(
except Exception as err: # pylint: disable=broad-except
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

6
ingestion/src/metadata/test_suite/validations/column/column_value_mean_to_be_between.py
View File

@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -62,9 +63,10 @@ def column_value_mean_to_be_between(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

6
ingestion/src/metadata/test_suite/validations/column/column_value_median_to_be_between.py
View File

@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -62,9 +63,10 @@ def column_value_median_to_be_between(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

6
ingestion/src/metadata/test_suite/validations/column/column_value_min_to_be_between.py
View File

@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -62,9 +63,10 @@ def column_value_min_to_be_between(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

9
ingestion/src/metadata/test_suite/validations/column/column_value_stddev_to_be_between.py
View File

@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -23,9 +24,6 @@ from metadata.generated.schema.tests.basic import (
TestCaseStatus,
TestResultValue,
)
from metadata.generated.schema.tests.column.columnValueStdDevToBeBetween import (
ColumnValueStdDevToBeBetween,
)
from metadata.generated.schema.tests.testCase import TestCase
from metadata.orm_profiler.metrics.registry import Metrics
from metadata.orm_profiler.profiler.runner import QueryRunner
@ -65,9 +63,10 @@ def column_value_stddev_to_be_between(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

10
ingestion/src/metadata/test_suite/validations/column/column_values_in_set.py
View File

@ -13,7 +13,7 @@
ColumnValuesToBeNotNull validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from ast import literal_eval
from datetime import datetime
@ -24,9 +24,6 @@ from metadata.generated.schema.tests.basic import (
TestCaseStatus,
TestResultValue,
)
from metadata.generated.schema.tests.column.columnValuesToBeInSet import (
ColumnValuesToBeInSet,
)
from metadata.generated.schema.tests.testCase import TestCase
from metadata.orm_profiler.metrics.core import add_props
from metadata.orm_profiler.metrics.registry import Metrics
@ -77,9 +74,10 @@ def column_values_in_set(
except Exception as err: # pylint: disable=broad-except
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

6
ingestion/src/metadata/test_suite/validations/column/column_values_length_to_be_between.py
View File

@ -14,6 +14,7 @@ ColumnValueLengthsToBeBetween validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -70,9 +71,10 @@ def column_value_length_to_be_between(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

17
ingestion/src/metadata/test_suite/validations/column/column_values_missing_count_to_be_equal.py
View File

@ -12,9 +12,9 @@
"""
ColumnValuesMissingCount validation implementation
"""
from ast import literal_eval
# pylint: disable=duplicate-code
import traceback
from ast import literal_eval
from datetime import datetime
from sqlalchemy import inspect
@ -24,9 +24,6 @@ from metadata.generated.schema.tests.basic import (
TestCaseStatus,
TestResultValue,
)
from metadata.generated.schema.tests.column.columnValuesMissingCountToBeEqual import (
ColumnValuesMissingCount,
)
from metadata.generated.schema.tests.testCase import TestCase
from metadata.orm_profiler.metrics.core import add_props
from metadata.orm_profiler.metrics.registry import Metrics
@ -69,9 +66,10 @@ def column_values_missing_count_to_be_equal(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,
@ -108,8 +106,9 @@ def column_values_missing_count_to_be_equal(
null_count_value_res += set_count_res
except Exception as err: # pylint: disable=broad-except
msg = f"Error computing {test_case.__class__.__name__} for {runner.table.__tablename__} - {err}"
logger.error(msg)
msg = f"Error computing {test_case.__class__.__name__} for {runner.table.__tablename__}: {err}"
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

7
ingestion/src/metadata/test_suite/validations/column/column_values_not_in_set.py
View File

@ -13,7 +13,7 @@
ColumnValuesToBeNotNull validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from ast import literal_eval
from datetime import datetime
@ -73,9 +73,10 @@ def column_values_not_in_set(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

6
ingestion/src/metadata/test_suite/validations/column/column_values_sum_to_be_between.py
View File

@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -62,9 +63,10 @@ def column_values_sum_to_be_between(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

9
ingestion/src/metadata/test_suite/validations/column/column_values_to_be_between.py
View File

@ -14,6 +14,7 @@ ColumnValuesToBeBetween validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -23,9 +24,6 @@ from metadata.generated.schema.tests.basic import (
TestCaseStatus,
TestResultValue,
)
from metadata.generated.schema.tests.column.columnValuesToBeBetween import (
ColumnValuesToBeBetween,
)
from metadata.generated.schema.tests.testCase import TestCase
from metadata.orm_profiler.metrics.registry import Metrics
from metadata.orm_profiler.profiler.runner import QueryRunner
@ -69,9 +67,10 @@ def column_values_to_be_between(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

9
ingestion/src/metadata/test_suite/validations/column/column_values_to_be_not_null.py
View File

@ -14,6 +14,7 @@ ColumnValuesToBeNotNull validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -23,9 +24,6 @@ from metadata.generated.schema.tests.basic import (
TestCaseStatus,
TestResultValue,
)
from metadata.generated.schema.tests.column.columnValuesToBeNotNull import (
ColumnValuesToBeNotNull,
)
from metadata.generated.schema.tests.testCase import TestCase
from metadata.orm_profiler.metrics.registry import Metrics
from metadata.orm_profiler.profiler.runner import QueryRunner
@ -65,9 +63,10 @@ def column_values_to_be_not_null(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

9
ingestion/src/metadata/test_suite/validations/column/column_values_to_be_unique.py
View File

@ -14,6 +14,7 @@ ColumnValuesToBeUnique validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -24,9 +25,6 @@ from metadata.generated.schema.tests.basic import (
TestCaseStatus,
TestResultValue,
)
from metadata.generated.schema.tests.column.columnValuesToBeUnique import (
ColumnValuesToBeUnique,
)
from metadata.generated.schema.tests.testCase import TestCase
from metadata.orm_profiler.metrics.registry import Metrics
from metadata.orm_profiler.profiler.runner import QueryRunner
@ -76,8 +74,9 @@ def column_values_to_be_unique(
unique_count_value_res = unique_count_value_dict.get(Metrics.UNIQUE_COUNT.name)
except Exception as err:
msg = f"Error computing {test_case.name.__root__} for {runner.table.__tablename__} - {err}"
logger.error(msg)
msg = f"Error computing {test_case.name.__root__} for {runner.table.__tablename__}: {err}"
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

10
ingestion/src/metadata/test_suite/validations/column/column_values_to_match_regex.py
View File

@ -13,7 +13,7 @@
ColumnValuesToBeNotNull validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -23,9 +23,6 @@ from metadata.generated.schema.tests.basic import (
TestCaseStatus,
TestResultValue,
)
from metadata.generated.schema.tests.column.columnValuesToMatchRegex import (
ColumnValuesToMatchRegex,
)
from metadata.generated.schema.tests.testCase import TestCase
from metadata.orm_profiler.metrics.core import add_props
from metadata.orm_profiler.metrics.registry import Metrics
@ -78,9 +75,10 @@ def column_values_to_match_regex(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

7
ingestion/src/metadata/test_suite/validations/column/column_values_to_not_match_regex.py
View File

@ -13,7 +13,7 @@
ColumnValuesToBeNotNull validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -75,9 +75,10 @@ def column_values_to_not_match_regex(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

6
ingestion/src/metadata/test_suite/validations/table/table_column_count_to_be_between.py
View File

@ -14,6 +14,7 @@ TableColumnCountToBeBetween validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -51,9 +52,10 @@ def table_column_count_to_be_between(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

6
ingestion/src/metadata/test_suite/validations/table/table_column_count_to_equal.py
View File

@ -14,6 +14,7 @@ TableColumnCountToEqual validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import inspect
@ -48,9 +49,10 @@ def table_column_count_to_equal(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

6
ingestion/src/metadata/test_suite/validations/table/table_column_name_to_exist.py
View File

@ -15,6 +15,7 @@ TableColumnCountToBeBetween validation implementation
# pylint: disable=duplicate-code
import reprlib
import traceback
from datetime import datetime
from typing import List
@ -65,9 +66,10 @@ def table_column_name_to_exist(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

6
ingestion/src/metadata/test_suite/validations/table/table_column_to_match_set.py
View File

@ -16,6 +16,7 @@ TableColumnCountToBeBetween validation implementation
import collections
import reprlib
import traceback
from datetime import datetime
from typing import List
@ -66,9 +67,10 @@ def table_column_to_match_set(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

6
ingestion/src/metadata/test_suite/validations/table/table_custom_sql_query.py
View File

@ -14,6 +14,7 @@ TableColumnCountToBeBetween validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from sqlalchemy import text
@ -60,9 +61,10 @@ def table_custom_sql_query(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

6
ingestion/src/metadata/test_suite/validations/table/table_row_count_to_be_between.py
View File

@ -14,6 +14,7 @@ TableRowCountToBeBetween validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from metadata.generated.schema.tests.basic import (
@ -50,9 +51,10 @@ def table_row_count_to_be_between(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,

6
ingestion/src/metadata/test_suite/validations/table/table_row_count_to_equal.py
View File

@ -14,6 +14,7 @@ TableRowCountToEqual validation implementation
"""
# pylint: disable=duplicate-code
import traceback
from datetime import datetime
from metadata.generated.schema.tests.basic import (
@ -50,9 +51,10 @@ def table_row_count_to_equal(
except Exception as err:
msg = (
f"Error computing {test_case.name} for {runner.table.__tablename__} - {err}"
f"Error computing {test_case.name} for {runner.table.__tablename__}: {err}"
)
logger.error(msg)
logger.debug(traceback.format_exc())
logger.warning(msg)
return TestCaseResult(
timestamp=execution_date,
testCaseStatus=TestCaseStatus.Aborted,