yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-08 05:26:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

added usage query to logs in case of error (#16455)

Browse Source
This commit is contained in:
Imri Paran 2024-05-29 19:14:03 +02:00 committed by GitHub
parent d55981adfd
commit c8dd9971b4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ingestion/src/metadata/ingestion/source/database/postgres/usage.py
View File

@ -39,9 +39,11 @@ class PostgresUsageSource(PostgresQueryParserSource, UsageSource):
"""
Process Query
"""
query = None
try:
query = self.get_sql_statement()
with get_connection(self.service_connection).connect() as conn:
rows = conn.execute(self.get_sql_statement())
rows = conn.execute(query)
queries = []
for row in rows:
row = dict(row)
@ -64,5 +66,9 @@ class PostgresUsageSource(PostgresQueryParserSource, UsageSource):
if queries:
yield TableQueries(queries=queries)
except Exception as err:
if query:
logger.debug(
f"###### USAGE QUERY #######\n{query}\n##########################"
)
logger.error(f"Source usage processing error - {err}")
logger.debug(traceback.format_exc())

16
ingestion/src/metadata/ingestion/source/database/usage_source.py
View File

@ -93,15 +93,15 @@ class UsageSource(QueryParserSource, ABC):
f"Scanning query logs for {(self.start + timedelta(days=days)).date()} - "
f"{(self.start + timedelta(days=days + 1)).date()}"
)
query = None
try:
query = self.get_sql_statement(
start_time=self.start + timedelta(days=days),
end_time=self.start + timedelta(days=days + 1),
)
for engine in self.get_engine():
with engine.connect() as conn:
rows = conn.execute(
self.get_sql_statement(
start_time=self.start + timedelta(days=days),
end_time=self.start + timedelta(days=days + 1),
)
)
rows = conn.execute(query)
queries = []
for row in rows:
row = dict(row)
@ -133,6 +133,10 @@ class UsageSource(QueryParserSource, ABC):
)
yield TableQueries(queries=queries)
except Exception as exc:
if query:
logger.debug(
f"###### USAGE QUERY #######\n{query}\n##########################"
)
logger.debug(traceback.format_exc())
logger.error(f"Source usage processing error: {exc}")