yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 09:58:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingestion/fivetran): Fix fivetran get connector jobs bug (#9975)

Browse Source
This commit is contained in:
Shubham Jagtap 2024-03-05 22:01:28 +05:30 committed by GitHub
parent a0efb9dc1d
commit 4d053f1ad3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py
View File

@ -132,7 +132,10 @@ class FivetranLogAPI:
# If no sync-end event log for this sync id that means sync is still in progress
continue
message_data = json.loads(sync_end_logs[sync_id][Constant.MESSAGE_DATA])
message_data = sync_end_logs[sync_id][Constant.MESSAGE_DATA]
if message_data is None:
continue
message_data = json.loads(message_data)
if isinstance(message_data, str):
# Sometimes message_data contains json string inside string
# Ex: '"{\"status\":\"SUCCESSFUL\"}"'

9
metadata-ingestion/tests/integration/fivetran/test_fivetran.py
View File

@ -95,6 +95,10 @@ def default_query_results(
"time_stamp": datetime.datetime(2023, 10, 3, 14, 35, 55, 401000),
"sync_id": "63c2fc85-600b-455f-9ba0-f576522465be",
},
{
"time_stamp": datetime.datetime(2023, 10, 3, 14, 37, 5, 403000),
"sync_id": "e773e1e9-c791-46f4-894f-8ch9b3dfc832",
},
]
elif query == fivetran_log_query.get_sync_end_logs_query("calendar_elected"):
return [
@ -113,6 +117,11 @@ def default_query_results(
"sync_id": "63c2fc85-600b-455f-9ba0-f576522465be",
"message_data": '"{\\"reason\\":\\"java.lang.RuntimeException: FATAL: too many connections for role \\\\\\"hxwraqld\\\\\\"\\",\\"taskType\\":\\"reconnect\\",\\"status\\":\\"FAILURE_WITH_TASK\\"}"',
},
{
"time_stamp": datetime.datetime(2023, 10, 3, 14, 37, 35, 478000),
"sync_id": "e773e1e9-c791-46f4-894f-8ch9b3dfc832",
"message_data": None,
},
]
# Unreachable code
raise Exception(f"Unknown query {query}")