yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-11 07:16:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add filter patterns in tableau (#4321)

Browse Source
This commit is contained in:
Ayush Shah 2022-04-21 17:39:48 +05:30 committed by GitHub
parent ced262e7c8
commit e4e1d4971b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 87 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
ingestion/src/metadata/ingestion/source/tableau.py
View File

@ -13,6 +13,7 @@ Tableau source module
""" """
import logging import logging
import traceback
import uuid import uuid
from typing import Iterable, List from typing import Iterable, List
@ -46,6 +47,7 @@ from metadata.ingestion.api.common import Entity
from metadata.ingestion.api.source import InvalidSourceException, Source, SourceStatus from metadata.ingestion.api.source import InvalidSourceException, Source, SourceStatus
from metadata.ingestion.models.table_metadata import Chart, Dashboard, DashboardOwner from metadata.ingestion.models.table_metadata import Chart, Dashboard, DashboardOwner
from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.utils.filters import filter_by_chart, filter_by_dashboard
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -80,6 +82,7 @@ class TableauSource(Source[Entity]):
self.metadata_config = metadata_config self.metadata_config = metadata_config
self.metadata = OpenMetadata(metadata_config) self.metadata = OpenMetadata(metadata_config)
self.connection_config = self.config.serviceConnection.__root__.config self.connection_config = self.config.serviceConnection.__root__.config
self.source_config = self.config.sourceConfig.config
self.client = self.tableau_client() self.client = self.tableau_client()
self.service = self.metadata.get_service_or_create( self.service = self.metadata.get_service_or_create(
entity=DashboardService, config=config entity=DashboardService, config=config
@ -125,9 +128,9 @@ class TableauSource(Source[Entity]):
env=self.connection_config.env, env=self.connection_config.env,
) )
conn.sign_in().json() conn.sign_in().json()
return conn
except Exception as err: # pylint: disable=broad-except except Exception as err: # pylint: disable=broad-except
logger.error("%s: %s", repr(err), err) logger.error("%s: %s", repr(err), err)
return conn
@classmethod @classmethod
def create(cls, config_dict: dict, metadata_config: OpenMetadataConnection): def create(cls, config_dict: dict, metadata_config: OpenMetadataConnection):
@ -195,8 +198,14 @@ class TableauSource(Source[Entity]):
def _get_tableau_dashboard(self) -> Dashboard: def _get_tableau_dashboard(self) -> Dashboard:
for index in range(len(self.dashboards["id"])): for index in range(len(self.dashboards["id"])):
try:
dashboard_id = self.dashboards["id"][index] dashboard_id = self.dashboards["id"][index]
dashboard_name = self.dashboards["name"][index] dashboard_name = self.dashboards["name"][index]
if filter_by_dashboard(
self.source_config.dashboardFilterPattern, dashboard_name
):
self.status.failure(dashboard_name, "Dashboard Pattern not allowed")
continue
dashboard_tag = self.dashboards["tags"][index] dashboard_tag = self.dashboards["tags"][index]
dashboard_url = self.dashboards["webpageUrl"][index] dashboard_url = self.dashboards["webpageUrl"][index]
datasource_list = ( datasource_list = (
@ -225,15 +234,25 @@ class TableauSource(Source[Entity]):
charts=dashboard_chart, charts=dashboard_chart,
tags=list(tag_labels), tags=list(tag_labels),
url=dashboard_url, url=dashboard_url,
service=EntityReference(id=self.service.id, type="dashboardService"), service=EntityReference(
last_modified=dateparser.parse(chart["updatedAt"]).timestamp() * 1000, id=self.service.id, type="dashboardService"
),
last_modified=dateparser.parse(chart["updatedAt"]).timestamp()
* 1000,
) )
if self.config.serviceName: if self.config.serviceName:
yield from self.get_lineage(datasource_list, dashboard_id) yield from self.get_lineage(datasource_list, dashboard_id)
except Exception as err:
logger.debug(traceback.format_exc())
logger.error(err)
def _get_tableau_charts(self): def _get_tableau_charts(self):
for index in range(len(self.all_dashboard_details["id"])): for index in range(len(self.all_dashboard_details["id"])):
try:
chart_name = self.all_dashboard_details["name"][index] chart_name = self.all_dashboard_details["name"][index]
if filter_by_chart(self.source_config.chartFilterPattern, chart_name):
self.status.failure(chart_name, "Chart Pattern not allowed")
continue
chart_id = self.all_dashboard_details["id"][index] chart_id = self.all_dashboard_details["id"][index]
chart_tags = self.all_dashboard_details["tags"][index] chart_tags = self.all_dashboard_details["tags"][index]
chart_type = self.all_dashboard_details["sheetType"][index] chart_type = self.all_dashboard_details["sheetType"][index]
@ -256,9 +275,15 @@ class TableauSource(Source[Entity]):
url=chart_url, url=chart_url,
owners=self.get_owner(chart_owner), owners=self.get_owner(chart_owner),
datasource_fqn=chart_datasource_fqn, datasource_fqn=chart_datasource_fqn,
last_modified=dateparser.parse(chart_last_modified).timestamp() * 1000, last_modified=dateparser.parse(chart_last_modified).timestamp()
service=EntityReference(id=self.service.id, type="dashboardService"), * 1000,
service=EntityReference(
id=self.service.id, type="dashboardService"
),
) )
except Exception as err:
logger.debug(traceback.format_exc())
logger.error(err)
def get_status(self) -> SourceStatus: def get_status(self) -> SourceStatus:
return self.status return self.status