From 682626ffe86bd53e12a35eec9d359838161dbba6 Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Wed, 15 Mar 2023 14:00:03 +0530 Subject: [PATCH] Added exception handling in dashboard sources (#10573) * added exception handling in dashboard * updated process owner logic * Added redash source for fixes --- .../source/dashboard/dashboard_service.py | 28 ++++++++++++++++++- .../dashboard/domodashboard/metadata.py | 19 +++---------- .../source/dashboard/looker/metadata.py | 12 -------- .../source/dashboard/powerbi/metadata.py | 8 +++++- .../source/dashboard/redash/metadata.py | 14 ---------- .../source/dashboard/superset/mixin.py | 10 ------- .../source/dashboard/tableau/metadata.py | 12 -------- 7 files changed, 38 insertions(+), 65 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py index 62dee591a94..7de9e774ebb 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py @@ -274,10 +274,36 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC): entity=DashboardService, config=config ) - def process_owner(self, dashboard_details: dict): # pylint: disable=unused-argument + def process_owner(self, dashboard_details): + try: + owner = self.get_owner_details( # pylint: disable=assignment-from-none + dashboard_details=dashboard_details + ) + if owner and self.source_config.overrideOwner: + self.metadata.patch_owner( + entity=Dashboard, + entity_id=self.context.dashboard.id, + owner=owner, + force=True, + ) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Error processing owner for {dashboard_details}: {exc}") + + def get_owner_details( # pylint: disable=useless-return + self, dashboard_details # pylint: disable=unused-argument + ) -> Optional[EntityReference]: + """Get dashboard owner + + Args: + dashboard_details: + Returns: + Optional[EntityReference] + """ logger.debug( f"Processing ownership is not supported for {self.service_connection.type.name}" ) + return None @staticmethod def _get_add_lineage_request( diff --git a/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py index e5c1715e90e..5de05ebed54 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py @@ -27,7 +27,6 @@ from metadata.generated.schema.api.data.createChart import CreateChartRequest from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.entity.data.chart import Chart -from metadata.generated.schema.entity.data.dashboard import Dashboard from metadata.generated.schema.entity.services.connections.dashboard.domoDashboardConnection import ( DomoDashboardConnection, ) @@ -95,8 +94,10 @@ class DomodashboardSource(DashboardServiceSource): def get_dashboard_details(self, dashboard: DomoDashboardDetails) -> dict: return dashboard - def get_owner_details(self, owners: List[DomoOwner]) -> Optional[EntityReference]: - for owner in owners: + def get_owner_details( + self, dashboard_details: DomoDashboardDetails + ) -> Optional[EntityReference]: + for owner in dashboard_details.owners: try: owner_details = self.client.users_get(owner.id) if owner_details.get("email"): @@ -112,18 +113,6 @@ class DomodashboardSource(DashboardServiceSource): ) return None - def process_owner( - self, dashboard_details: DomoDashboardDetails - ) -> Optional[Dashboard]: - owner = self.get_owner_details(owners=dashboard_details.owners) - if owner and self.source_config.overrideOwner: - self.metadata.patch_owner( - entity=Dashboard, - entity_id=self.context.dashboard.id, - owner=owner, - force=True, - ) - def yield_dashboard( self, dashboard_details: DomoDashboardDetails ) -> Iterable[CreateDashboardRequest]: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py index 39320aca113..776e176e821 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py @@ -182,18 +182,6 @@ class LookerSource(DashboardServiceSource): return self._owners_ref.get(dashboard_details.user_id) - def process_owner( - self, dashboard_details: LookerDashboard - ) -> Optional[MetadataDashboard]: - owner = self.get_owner_details(dashboard_details=dashboard_details) - if owner and self.source_config.overrideOwner: - self.metadata.patch_owner( - entity=MetadataDashboard, - entity_id=self.context.dashboard.id, - owner=owner, - force=True, - ) - def yield_dashboard( self, dashboard_details: LookerDashboard ) -> CreateDashboardRequest: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py index e1d8dcd4570..8a0865caf72 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py @@ -91,7 +91,13 @@ class PowerbiSource(DashboardServiceSource): response = self.client.fetch_workspace_scan_result( scan_id=workspace_scan_id ) - self.workspace_data.extend(response.get("workspaces")) + self.workspace_data.extend( + [ + active_workspace + for active_workspace in response.get("workspaces") + if active_workspace.get("state") == "Active" + ] + ) else: logger.error("Error in fetching dashboards and charts") count += 1 diff --git a/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py index e478f808639..582504d6b73 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py @@ -173,20 +173,6 @@ class RedashSource(DashboardServiceSource): return EntityReference(id=user.id.__root__, type="user") return None - def process_owner(self, dashboard_details) -> Optional[LineageDashboard]: - try: - owner = self.get_owner_details(dashboard_details=dashboard_details) - if owner and self.source_config.overrideOwner: - self.metadata.patch_owner( - entity=LineageDashboard, - entity_id=self.context.dashboard.id, - owner=owner, - force=True, - ) - except Exception as exc: - logger.debug(traceback.format_exc()) - logger.warning(f"Error processing owner for {dashboard_details}: {exc}") - def get_dashboard_url(self, dashboard_details: dict) -> str: if version.parse(self.service_connection.redashVersion) > version.parse( INCOMPATIBLE_REDASH_VERSION diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py index 49e1242a637..e9c626d01d9 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py @@ -88,16 +88,6 @@ class SupersetSourceMixin(DashboardServiceSource): return None - def process_owner(self, dashboard_details: dict) -> Optional[Lineage_Dashboard]: - owner = self.get_owner_details(dashboard_details=dashboard_details) - if owner and self.source_config.overrideOwner: - self.metadata.patch_owner( - self.metadata, - entity=Lineage_Dashboard, - entity_id=self.context.dashboard.id, - force=True, - ) - def get_owner_details(self, dashboard_details: dict) -> EntityReference: for owner in dashboard_details.get("owners", []): user = self._get_user_by_email(owner["email"]) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py index 6e04a514cd4..eb39506cdbc 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py @@ -251,18 +251,6 @@ class TableauSource(DashboardServiceSource): return EntityReference(id=user.id.__root__, type="user") return None - def process_owner( - self, dashboard_details: TableauDashboard - ) -> Optional[LineageDashboard]: - owner = self.get_owner_details(dashboard_details=dashboard_details) - if owner and self.source_config.overrideOwner: - self.metadata.patch_owner( - entity=LineageDashboard, - entity_id=self.context.dashboard.id, - owner=owner, - force=True, - ) - def yield_tag(self, *_, **__) -> OMetaTagAndClassification: """ Fetch Dashboard Tags