yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-23 08:14:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/logging): fix excessive ingestion logging (#10735)

Browse Source
This commit is contained in:
pie1nthesky 2024-06-19 13:57:39 +03:00 committed by GitHub
parent 750aab9a51
commit dfd5bcf5dd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
View File

@ -21,7 +21,7 @@ from typing import (
) )
import sqlalchemy.dialects.postgresql.base import sqlalchemy.dialects.postgresql.base
from sqlalchemy import create_engine, inspect from sqlalchemy import create_engine, inspect, log as sqlalchemy_log
from sqlalchemy.engine.reflection import Inspector from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.row import LegacyRow from sqlalchemy.engine.row import LegacyRow
from sqlalchemy.exc import ProgrammingError from sqlalchemy.exc import ProgrammingError
@ -536,6 +536,9 @@ class SQLAlchemySource(StatefulIngestionSourceBase, TestableSource):
if logger.isEnabledFor(logging.DEBUG): if logger.isEnabledFor(logging.DEBUG):
# If debug logging is enabled, we also want to echo each SQL query issued. # If debug logging is enabled, we also want to echo each SQL query issued.
sql_config.options.setdefault("echo", True) sql_config.options.setdefault("echo", True)
# Patch to avoid duplicate logging
# Known issue with sqlalchemy https://stackoverflow.com/questions/60804288/pycharm-duplicated-log-for-sqlalchemy-echo-true
sqlalchemy_log._add_default_handler = lambda x: None # type: ignore
# Extra default SQLAlchemy option for better connection pooling and threading. # Extra default SQLAlchemy option for better connection pooling and threading.
# https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool.params.max_overflow # https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool.params.max_overflow

4
metadata-ingestion/src/datahub/utilities/logging_manager.py
View File

@ -149,9 +149,9 @@ class _DatahubLogFilter(logging.Filter):
return record.levelno >= logging.INFO return record.levelno >= logging.INFO
else: else:
if self.debug: if self.debug:
return record.levelno >= logging.WARNING
else:
return record.levelno >= logging.INFO return record.levelno >= logging.INFO
else:
return record.levelno >= logging.WARNING
class _LogBuffer: class _LogBuffer: