yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-08 23:14:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: Fix Datetime Conversion issue in usage/lineage (#17380)

Browse Source 
* MINOR: Fix Datetime Conversion issue in usage/lineage

* Undo mssql specific fixes

* fixed datetime conversion in mssql

* fixed datetime conversion in oracle
This commit is contained in:
Suman Maharana 2024-08-13 17:34:50 +05:30 committed by GitHub
parent 5e32c2aa78
commit feab12422b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 6 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ingestion/src/metadata/ingestion/source/database/mssql/lineage.py
View File

@ -26,10 +26,6 @@ from metadata.ingestion.source.database.mssql.utils import (
class MssqlLineageSource(MssqlQueryParserSource, LineageSource): class MssqlLineageSource(MssqlQueryParserSource, LineageSource):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.start = self.start.replace(tzinfo=None)
self.end = self.end.replace(tzinfo=None)
sql_stmt = MSSQL_SQL_STATEMENT sql_stmt = MSSQL_SQL_STATEMENT

2
ingestion/src/metadata/ingestion/source/database/mssql/metadata.py
View File

@ -224,7 +224,7 @@ class MssqlSource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource):
current_datetime_format = MSSQL_DATEFORMAT_DATETIME_MAP.get( current_datetime_format = MSSQL_DATEFORMAT_DATETIME_MAP.get(
server_date_format, DEFAULT_DATETIME_FORMAT server_date_format, DEFAULT_DATETIME_FORMAT
) )
start = start.replace(tzinfo=None).strftime(current_datetime_format) start = start.strftime(current_datetime_format)
query = MSSQL_GET_STORED_PROCEDURE_QUERIES.format( query = MSSQL_GET_STORED_PROCEDURE_QUERIES.format(
start_date=start, start_date=start,
) )

4
ingestion/src/metadata/ingestion/source/database/mssql/usage.py
View File

@ -61,8 +61,8 @@ class MssqlUsageSource(MssqlQueryParserSource, UsageSource):
Override if we have specific parameters Override if we have specific parameters
""" """
return self.sql_stmt.format( return self.sql_stmt.format(
start_time=start_time.replace(tzinfo=None).strftime(self.dt_format), start_time=start_time.strftime(self.dt_format),
end_time=end_time.replace(tzinfo=None).strftime(self.dt_format), end_time=end_time.strftime(self.dt_format),
filters=self.get_filters(), filters=self.get_filters(),
result_limit=self.source_config.resultLimit, result_limit=self.source_config.resultLimit,
) )

4
ingestion/src/metadata/ingestion/source/database/oracle/queries.py
View File

@ -104,7 +104,7 @@ WITH SP_HISTORY AS (SELECT
PARSING_SCHEMA_NAME as user_name PARSING_SCHEMA_NAME as user_name
FROM gv$sql FROM gv$sql
WHERE UPPER(sql_text) LIKE '%%CALL%%' or UPPER(sql_text) LIKE '%%BEGIN%%' WHERE UPPER(sql_text) LIKE '%%CALL%%' or UPPER(sql_text) LIKE '%%BEGIN%%'
AND TO_TIMESTAMP(FIRST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS') >= TO_TIMESTAMP_TZ('{start_date}', 'YYYY-MM-DD HH24:MI:SS+TZH:TZM') AND TO_TIMESTAMP(FIRST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS') >= TO_TIMESTAMP('{start_date}', 'YYYY-MM-DD HH24:MI:SS')
), ),
Q_HISTORY AS (SELECT Q_HISTORY AS (SELECT
sql_text AS query_text, sql_text AS query_text,
@ -124,7 +124,7 @@ WITH SP_HISTORY AS (SELECT
AND SQL_FULLTEXT NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' AND SQL_FULLTEXT NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
AND SQL_FULLTEXT NOT LIKE '/* {{"app": "dbt", %%}} */%%' AND SQL_FULLTEXT NOT LIKE '/* {{"app": "dbt", %%}} */%%'
AND TO_TIMESTAMP(FIRST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS') AND TO_TIMESTAMP(FIRST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS')
>= TO_TIMESTAMP_TZ('{start_date}', 'YYYY-MM-DD HH24:MI:SS+TZH:TZM') >= TO_TIMESTAMP('{start_date}', 'YYYY-MM-DD HH24:MI:SS')
) )
SELECT SELECT
Q.QUERY_TYPE AS QUERY_TYPE, Q.QUERY_TYPE AS QUERY_TYPE,

2
ingestion/src/metadata/utils/helpers.py
View File

@ -123,7 +123,7 @@ def get_start_and_end(duration: int = 0) -> Tuple[datetime, datetime]:
Method to return start and end time based on duration Method to return start and end time based on duration
""" """
today = datetime.now(timezone.utc) today = datetime.now(timezone.utc).replace(tzinfo=None)
start = (today + timedelta(0 - duration)).replace( start = (today + timedelta(0 - duration)).replace(
hour=0, minute=0, second=0, microsecond=0 hour=0, minute=0, second=0, microsecond=0
) )