diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index 3091791551..cd54ab7744 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -21,7 +21,7 @@ from typing import ( ) import sqlalchemy.dialects.postgresql.base -from sqlalchemy import create_engine, inspect +from sqlalchemy import create_engine, inspect, log as sqlalchemy_log from sqlalchemy.engine.reflection import Inspector from sqlalchemy.engine.row import LegacyRow from sqlalchemy.exc import ProgrammingError @@ -536,6 +536,9 @@ class SQLAlchemySource(StatefulIngestionSourceBase, TestableSource): if logger.isEnabledFor(logging.DEBUG): # If debug logging is enabled, we also want to echo each SQL query issued. sql_config.options.setdefault("echo", True) + # Patch to avoid duplicate logging + # Known issue with sqlalchemy https://stackoverflow.com/questions/60804288/pycharm-duplicated-log-for-sqlalchemy-echo-true + sqlalchemy_log._add_default_handler = lambda x: None # type: ignore # Extra default SQLAlchemy option for better connection pooling and threading. # https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool.params.max_overflow diff --git a/metadata-ingestion/src/datahub/utilities/logging_manager.py b/metadata-ingestion/src/datahub/utilities/logging_manager.py index e71e695391..64383745eb 100644 --- a/metadata-ingestion/src/datahub/utilities/logging_manager.py +++ b/metadata-ingestion/src/datahub/utilities/logging_manager.py @@ -149,9 +149,9 @@ class _DatahubLogFilter(logging.Filter): return record.levelno >= logging.INFO else: if self.debug: - return record.levelno >= logging.WARNING - else: return record.levelno >= logging.INFO + else: + return record.levelno >= logging.WARNING class _LogBuffer: