yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-08 07:23:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): metabase - fix the datetime parsing issue (#3831)

Browse Source 
fixes #3803
This commit is contained in:
Ravindra Lanka 2022-01-05 16:34:05 -08:00 committed by GitHub
parent 480ad24282
commit 9f80e5487d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 18 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
metadata-ingestion/src/datahub/ingestion/source/metabase.py
View File

@ -1,3 +1,4 @@
from datetime import datetime
from functools import lru_cache from functools import lru_cache
from typing import Dict, Iterable, Optional from typing import Dict, Iterable, Optional
@ -131,6 +132,17 @@ class MetabaseSource(Source):
reason=f"Unable to retrieve dashboards. " f"Reason: {str(http_error)}", reason=f"Unable to retrieve dashboards. " f"Reason: {str(http_error)}",
) )
@staticmethod
def get_timestamp_millis_from_ts_string(ts_str: str) -> int:
"""
Converts the given timestamp string to milliseconds. If parsing fails,
returns the utc-now in milliseconds.
"""
try:
return int(dp.parse(ts_str).timestamp() * 1000)
except (dp.ParserError, OverflowError):
return int(datetime.utcnow().timestamp() * 1000)
def construct_dashboard_from_api_data( def construct_dashboard_from_api_data(
self, dashboard_info: dict self, dashboard_info: dict
) -> Optional[DashboardSnapshot]: ) -> Optional[DashboardSnapshot]:
@ -157,8 +169,8 @@ class MetabaseSource(Source):
) )
last_edit_by = dashboard_details.get("last-edit-info") or {} last_edit_by = dashboard_details.get("last-edit-info") or {}
modified_actor = builder.make_user_urn(last_edit_by.get("email", "unknown")) modified_actor = builder.make_user_urn(last_edit_by.get("email", "unknown"))
modified_ts = int( modified_ts = self.get_timestamp_millis_from_ts_string(
dp.parse(f"{last_edit_by.get('timestamp', 'now')}").timestamp() * 1000 f"{last_edit_by.get('timestamp')}"
) )
title = dashboard_details.get("name", "") or "" title = dashboard_details.get("name", "") or ""
description = dashboard_details.get("description", "") or "" description = dashboard_details.get("description", "") or ""
@ -261,8 +273,8 @@ class MetabaseSource(Source):
last_edit_by = card_details.get("last-edit-info") or {} last_edit_by = card_details.get("last-edit-info") or {}
modified_actor = builder.make_user_urn(last_edit_by.get("email", "unknown")) modified_actor = builder.make_user_urn(last_edit_by.get("email", "unknown"))
modified_ts = int( modified_ts = self.get_timestamp_millis_from_ts_string(
dp.parse(f"{last_edit_by.get('timestamp', 'now')}").timestamp() * 1000 f"{last_edit_by.get('timestamp')}"
) )
last_modified = ChangeAuditStamps( last_modified = ChangeAuditStamps(
created=AuditStamp(time=modified_ts, actor=modified_actor), created=AuditStamp(time=modified_ts, actor=modified_actor),

5
metadata-ingestion/tests/integration/metabase/setup/card_2.json
View File

@ -100,8 +100,7 @@
"id": 1, "id": 1,
"email": "admin@metabase.com", "email": "admin@metabase.com",
"first_name": "FirstName", "first_name": "FirstName",
"last_name": "LastName", "last_name": "LastName"
"timestamp": "2021-12-13T17:48:37.11"
}, },
"visualization_settings": { "visualization_settings": {
"graph.series_labels": ["number"], "graph.series_labels": ["number"],
@ -112,4 +111,4 @@
"collection": null, "collection": null,
"created_at": "2021-12-13T17:48:37.102", "created_at": "2021-12-13T17:48:37.102",
"public_uuid": null "public_uuid": null
} }