diff --git a/ingestion/src/metadata/cli/docker.py b/ingestion/src/metadata/cli/docker.py index 6f5152276b6..8f592d12693 100644 --- a/ingestion/src/metadata/cli/docker.py +++ b/ingestion/src/metadata/cli/docker.py @@ -185,7 +185,6 @@ def run_docker( ) except Exception as err: logger.debug(traceback.format_exc()) - logger.debug(traceback.print_exc()) click.secho(str(err), fg="red") diff --git a/ingestion/src/metadata/ingestion/ometa/auth_provider.py b/ingestion/src/metadata/ingestion/ometa/auth_provider.py index 0b955a3f7d6..799af931f3f 100644 --- a/ingestion/src/metadata/ingestion/ometa/auth_provider.py +++ b/ingestion/src/metadata/ingestion/ometa/auth_provider.py @@ -243,7 +243,7 @@ class OktaAuthenticationProvider(AuthenticationProvider): self.expiry = response_dict.get(EXPIRY) except Exception as err: - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(err) sys.exit() diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py index f486ec5cfa3..a90cabda8b4 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py @@ -210,7 +210,7 @@ class OMetaLineageMixin(Generic[T]): logger.info(f"Successfully added Lineage {created_lineage}") except Exception as err: - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(err) def ingest_lineage_by_query( diff --git a/ingestion/src/metadata/ingestion/processor/query_parser.py b/ingestion/src/metadata/ingestion/processor/query_parser.py index 56535479309..5928749ca82 100644 --- a/ingestion/src/metadata/ingestion/processor/query_parser.py +++ b/ingestion/src/metadata/ingestion/processor/query_parser.py @@ -13,6 +13,7 @@ Query parser implementation """ import datetime +import traceback from typing import Optional from sql_metadata import Parser @@ -91,7 +92,9 @@ class QueryParserProcessor(Processor): ) # pylint: disable=broad-except except Exception as err: - logger.debug(record.sql) + if hasattr(record, "sql"): + logger.debug(record.sql) + logger.debug(traceback.format_exc()) logger.error(err) query_parser_data = None diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch.py b/ingestion/src/metadata/ingestion/sink/elasticsearch.py index a55174b733a..6a9ee2418d3 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch.py @@ -266,7 +266,7 @@ class ElasticsearchSink(Sink[Entity]): self.status.records_written(record.name.__root__) except Exception as e: logger.error(f"Failed to index entity {record} due to {e}") - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) def _create_table_es_doc(self, table: Table): fqdn = table.fullyQualifiedName.__root__ diff --git a/ingestion/src/metadata/ingestion/sink/metadata_rest.py b/ingestion/src/metadata/ingestion/sink/metadata_rest.py index b8f65310cfc..90c9690cb9b 100644 --- a/ingestion/src/metadata/ingestion/sink/metadata_rest.py +++ b/ingestion/src/metadata/ingestion/sink/metadata_rest.py @@ -252,7 +252,7 @@ class MetadataRestSink(Sink[Entity]): db_schema_and_table.database.name.__root__, ) ) - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(err) self.status.failure(f"Table: {db_schema_and_table.table.name.__root__}") @@ -378,7 +378,7 @@ class MetadataRestSink(Sink[Entity]): except (APIError, ValidationError) as err: logger.error(f"Failed to ingest Policy {ometa_policy.policy.name}") logger.error(err) - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) self.status.failure(f"Policy: {ometa_policy.policy.name}") def _create_location(self, location: Location) -> Location: @@ -420,7 +420,6 @@ class MetadataRestSink(Sink[Entity]): return role except Exception as err: logger.debug(traceback.format_exc()) - logger.debug(traceback.print_exc()) logger.error(err) def _create_team(self, create_team: CreateTeamRequest) -> Team: @@ -430,7 +429,6 @@ class MetadataRestSink(Sink[Entity]): return team except Exception as err: logger.debug(traceback.format_exc()) - logger.debug(traceback.print_exc()) logger.error(err) def write_users(self, record: OMetaUserProfile): @@ -494,7 +492,6 @@ class MetadataRestSink(Sink[Entity]): logger.info("User: {}".format(user.displayName)) except Exception as err: logger.debug(traceback.format_exc()) - logger.debug(traceback.print_exc()) logger.error(err) def delete_table(self, record: DeleteTable): @@ -505,7 +502,6 @@ class MetadataRestSink(Sink[Entity]): ) except Exception as err: logger.debug(traceback.format_exc()) - logger.debug(traceback.print_exc()) logger.error(err) def write_table_tests(self, record: OMetaTableTest) -> None: @@ -532,7 +528,6 @@ class MetadataRestSink(Sink[Entity]): self.status.records_written(f"Table Tests: {record.table_name}.{test}") except Exception as err: logger.debug(traceback.format_exc()) - logger.debug(traceback.print_exc()) logger.error(err) def create_lineage_via_es(self, db_schema_and_table, db_schema, db): diff --git a/ingestion/src/metadata/ingestion/source/dynamodb.py b/ingestion/src/metadata/ingestion/source/dynamodb.py index 46b6fedd900..6c97cc8de3f 100644 --- a/ingestion/src/metadata/ingestion/source/dynamodb.py +++ b/ingestion/src/metadata/ingestion/source/dynamodb.py @@ -65,7 +65,6 @@ class DynamodbSource(Source[Entity]): yield from self.ingest_tables() except Exception as err: logger.debug(traceback.format_exc()) - logger.debug(traceback.print_exc()) logger.error(err) def ingest_tables(self, next_tables_token=None) -> Iterable[OMetaDatabaseAndTable]: @@ -107,7 +106,7 @@ class DynamodbSource(Source[Entity]): yield table_and_db except Exception as err: logger.debug(traceback.format_exc()) - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(err) def get_columns(self, column_data): @@ -127,7 +126,7 @@ class DynamodbSource(Source[Entity]): yield Column(**parsed_string) except Exception as err: logger.debug(traceback.format_exc()) - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(err) def close(self): diff --git a/ingestion/src/metadata/ingestion/source/glue.py b/ingestion/src/metadata/ingestion/source/glue.py index 8de4278e2df..87de0a97919 100644 --- a/ingestion/src/metadata/ingestion/source/glue.py +++ b/ingestion/src/metadata/ingestion/source/glue.py @@ -226,7 +226,6 @@ class GlueSource(Source[Entity]): yield from self.ingest_tables(glue_resp["NextToken"]) except Exception as err: logger.debug(traceback.format_exc()) - logger.debug(traceback.print_exc()) logger.error(err) def get_downstream_tasks(self, task_unique_id, tasks): @@ -283,7 +282,6 @@ class GlueSource(Source[Entity]): yield pipeline_ev except Exception as err: logger.debug(traceback.format_exc()) - logger.debug(traceback.print_exc()) logger.error(err) def close(self): diff --git a/ingestion/src/metadata/ingestion/source/looker.py b/ingestion/src/metadata/ingestion/source/looker.py index a05d61fb3b2..9c51f407a22 100644 --- a/ingestion/src/metadata/ingestion/source/looker.py +++ b/ingestion/src/metadata/ingestion/source/looker.py @@ -143,7 +143,7 @@ class LookerSource(Source[Entity]): try: yield from self._get_dashboard_elements(iter_chart) except Exception as err: - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(err) yield Dashboard( name=dashboard.id, diff --git a/ingestion/src/metadata/ingestion/source/metabase.py b/ingestion/src/metadata/ingestion/source/metabase.py index 84d59b5ce04..06e7228d762 100644 --- a/ingestion/src/metadata/ingestion/source/metabase.py +++ b/ingestion/src/metadata/ingestion/source/metabase.py @@ -158,7 +158,7 @@ class MetabaseSource(Source[Entity]): self.status.scanned(chart_details["name"]) except Exception as err: # pylint: disable=broad-except logger.error(repr(err)) - traceback.print_exc() + logger.debug(traceback.format_exc()) continue def get_dashboards(self): @@ -233,7 +233,7 @@ class MetabaseSource(Source[Entity]): ) yield lineage except Exception as err: # pylint: disable=broad-except,unused-variable - logger.error(traceback.print_exc()) + logger.error(traceback.format_exc()) def req_get(self, path): """Send get request method diff --git a/ingestion/src/metadata/ingestion/source/powerbi.py b/ingestion/src/metadata/ingestion/source/powerbi.py index 19c0b207587..b8c999f50a2 100644 --- a/ingestion/src/metadata/ingestion/source/powerbi.py +++ b/ingestion/src/metadata/ingestion/source/powerbi.py @@ -128,7 +128,7 @@ class PowerbiSource(Source[Entity]): self.charts.append(chart["id"]) self.status.scanned(chart["title"]) except Exception as err: # pylint: disable=broad-except - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(repr(err)) self.status.failure(chart["title"], err) @@ -165,7 +165,7 @@ class PowerbiSource(Source[Entity]): ), ) except Exception as err: - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(err) self.status.failure(dashboard_details["displayName"], err) diff --git a/ingestion/src/metadata/ingestion/source/sample_data.py b/ingestion/src/metadata/ingestion/source/sample_data.py index f084e442c99..d70bf728fbf 100644 --- a/ingestion/src/metadata/ingestion/source/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/sample_data.py @@ -473,7 +473,7 @@ class SampleDataSource(Source[Entity]): } sql_object["user"] = EntityReference(**user_dict, type="user") except Exception as err: - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.debug(err) table_metadata = Table(**table) table_and_db = OMetaDatabaseAndTable( @@ -589,7 +589,7 @@ class SampleDataSource(Source[Entity]): ) yield model_ev except Exception as err: - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(err) def ingest_users(self) -> Iterable[OMetaUserProfile]: diff --git a/ingestion/src/metadata/ingestion/source/snowflake_usage.py b/ingestion/src/metadata/ingestion/source/snowflake_usage.py index 1cea148be5d..6e7970af6f1 100644 --- a/ingestion/src/metadata/ingestion/source/snowflake_usage.py +++ b/ingestion/src/metadata/ingestion/source/snowflake_usage.py @@ -119,7 +119,7 @@ class SnowflakeUsageSource(Source[TableQuery]): self.report.scanned(f"{row['database_name']}") yield table_query except Exception as err: - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.debug(repr(err)) def get_report(self): diff --git a/ingestion/src/metadata/ingestion/source/sql_source.py b/ingestion/src/metadata/ingestion/source/sql_source.py index 68edf853f45..bd05301082d 100644 --- a/ingestion/src/metadata/ingestion/source/sql_source.py +++ b/ingestion/src/metadata/ingestion/source/sql_source.py @@ -184,7 +184,7 @@ class SQLSource(Source[OMetaDatabaseAndTable]): self.source_config.enableDataProfiler = False except Exception as exc: # pylint: disable=broad-except - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.debug(f"Error running ingestion profiler {repr(exc)}") return None @@ -359,7 +359,7 @@ class SQLSource(Source[OMetaDatabaseAndTable]): yield table_schema_and_db except Exception as err: - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(err) self.status.failures.append( "{}.{}".format(self.config.serviceName, table_name) @@ -501,7 +501,7 @@ class SQLSource(Source[OMetaDatabaseAndTable]): model_fqdn = f"{schema}.{model_name}".lower() self.data_models[model_fqdn] = model except Exception as err: - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(err) def _parse_data_model_upstream(self, mnode): @@ -771,12 +771,12 @@ class SQLSource(Source[OMetaDatabaseAndTable]): ) ] except Exception as err: - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(err) om_column = col_dict except Exception as err: - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(f"{err} : {column}") continue table_columns.append(om_column) diff --git a/ingestion/src/metadata/ingestion/source/superset.py b/ingestion/src/metadata/ingestion/source/superset.py index a47afc6a95b..4ea2a77f313 100644 --- a/ingestion/src/metadata/ingestion/source/superset.py +++ b/ingestion/src/metadata/ingestion/source/superset.py @@ -301,7 +301,7 @@ class SupersetSource(Source[Entity]): yield lineage except Exception as err: - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) logger.error(err) # pylint: disable=too-many-locals diff --git a/ingestion/src/metadata/orm_profiler/profiler/core.py b/ingestion/src/metadata/orm_profiler/profiler/core.py index dcd2199a51e..04a92afb8e5 100644 --- a/ingestion/src/metadata/orm_profiler/profiler/core.py +++ b/ingestion/src/metadata/orm_profiler/profiler/core.py @@ -289,7 +289,7 @@ class Profiler(Generic[MetricType]): except (TimeoutError, Exception) as err: # pylint: disable=broad-except print(err) - print(traceback.format_exc()) + logger.debug(traceback.format_exc()) logger.error( f"Error computing query metric {metric.name()} for {self.table.__tablename__}.{col.name} - {err}" ) diff --git a/ingestion/src/metadata/orm_profiler/sink/metadata_rest.py b/ingestion/src/metadata/orm_profiler/sink/metadata_rest.py index 1122c067299..9b38fe7335a 100644 --- a/ingestion/src/metadata/orm_profiler/sink/metadata_rest.py +++ b/ingestion/src/metadata/orm_profiler/sink/metadata_rest.py @@ -91,5 +91,5 @@ class MetadataRestSink(Sink[Entity]): logger.error( f"Failed to sink profiler & test data for {record.table.fullyQualifiedName.__root__} - {err}" ) - logger.debug(traceback.print_exc()) + logger.debug(traceback.format_exc()) self.status.failure(f"Table: {record.table.fullyQualifiedName.__root__}")