diff --git a/metadata-ingestion/src/datahub/ingestion/source/metabase.py b/metadata-ingestion/src/datahub/ingestion/source/metabase.py index 87331addf1..39be8747a1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metabase.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metabase.py @@ -1,3 +1,4 @@ +from datetime import datetime from functools import lru_cache from typing import Dict, Iterable, Optional @@ -131,6 +132,17 @@ class MetabaseSource(Source): reason=f"Unable to retrieve dashboards. " f"Reason: {str(http_error)}", ) + @staticmethod + def get_timestamp_millis_from_ts_string(ts_str: str) -> int: + """ + Converts the given timestamp string to milliseconds. If parsing fails, + returns the utc-now in milliseconds. + """ + try: + return int(dp.parse(ts_str).timestamp() * 1000) + except (dp.ParserError, OverflowError): + return int(datetime.utcnow().timestamp() * 1000) + def construct_dashboard_from_api_data( self, dashboard_info: dict ) -> Optional[DashboardSnapshot]: @@ -157,8 +169,8 @@ class MetabaseSource(Source): ) last_edit_by = dashboard_details.get("last-edit-info") or {} modified_actor = builder.make_user_urn(last_edit_by.get("email", "unknown")) - modified_ts = int( - dp.parse(f"{last_edit_by.get('timestamp', 'now')}").timestamp() * 1000 + modified_ts = self.get_timestamp_millis_from_ts_string( + f"{last_edit_by.get('timestamp')}" ) title = dashboard_details.get("name", "") or "" description = dashboard_details.get("description", "") or "" @@ -261,8 +273,8 @@ class MetabaseSource(Source): last_edit_by = card_details.get("last-edit-info") or {} modified_actor = builder.make_user_urn(last_edit_by.get("email", "unknown")) - modified_ts = int( - dp.parse(f"{last_edit_by.get('timestamp', 'now')}").timestamp() * 1000 + modified_ts = self.get_timestamp_millis_from_ts_string( + f"{last_edit_by.get('timestamp')}" ) last_modified = ChangeAuditStamps( created=AuditStamp(time=modified_ts, actor=modified_actor), diff --git a/metadata-ingestion/tests/integration/metabase/setup/card_2.json b/metadata-ingestion/tests/integration/metabase/setup/card_2.json index 979178cd24..7c18cd681c 100644 --- a/metadata-ingestion/tests/integration/metabase/setup/card_2.json +++ b/metadata-ingestion/tests/integration/metabase/setup/card_2.json @@ -100,8 +100,7 @@ "id": 1, "email": "admin@metabase.com", "first_name": "FirstName", - "last_name": "LastName", - "timestamp": "2021-12-13T17:48:37.11" + "last_name": "LastName" }, "visualization_settings": { "graph.series_labels": ["number"], @@ -112,4 +111,4 @@ "collection": null, "created_at": "2021-12-13T17:48:37.102", "public_uuid": null -} \ No newline at end of file +}