diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/dashboard/tableauConnection.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/dashboard/tableauConnection.json index 4483268aa11..ccba18e5237 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/dashboard/tableauConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/dashboard/tableauConnection.json @@ -19,7 +19,7 @@ "$ref": "#/definitions/tableauType", "default": "Tableau" }, - "server": { + "hostPort": { "description": "Tableau Server", "type": "string" }, @@ -40,11 +40,6 @@ "description": "Tableau Site Name", "type": "string" }, - "siteURL": { - "description": "Tableau Site URL", - "type": "string", - "format": "uri" - }, "personalAccessTokenName": { "description": "Personal Access Token Name", "type": "string" @@ -53,6 +48,10 @@ "description": "Personal Access Token Secret", "type": "string" }, + "env": { + "description": "Tableau Environment Name", + "type": "string" + }, "supportedPipelineTypes": { "description": "Supported Metadata Extraction Pipelines.", "type": "string", diff --git a/ingestion/examples/workflows/tableau.json b/ingestion/examples/workflows/tableau.json index 9f06899c2d4..18d45c809a9 100644 --- a/ingestion/examples/workflows/tableau.json +++ b/ingestion/examples/workflows/tableau.json @@ -1,28 +1,30 @@ { "source": { "type": "tableau", - "config": { - "personal_access_token_secret": "personal_access_token_secret", - "personal_access_token_name": "personal_access_token_name", - "username": "username", - "password": "password", - "service_name": "local_tableau", - "server": "server_address", - "site_name": "site_name", - "site_url": "site_url", - "api_version": "api version", - "env": "env" - } + "serviceName": "local_tableau", + "serviceConnection": { + "config": { + "type": "Tableau", + "username": "username", + "password": "password", + "env": "tableau_prod", + "hostPort": "localhost", + "siteName": "site_name", + "apiVersion": "api_version", + "personalAccessTokenName": "personal_access_token_name", + "personalAccessTokenSecret": "personal_access_token_secret" + } + }, + "sourceConfig": {"config": {"enableDataProfiler": false}} }, "sink": { "type": "metadata-rest", "config": {} }, - "metadata_server": { - "type": "metadata-server", - "config": { - "api_endpoint": "http://localhost:8585/api", - "auth_provider_type": "no-auth" + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "no-auth" } } } diff --git a/ingestion/src/metadata/ingestion/source/tableau.py b/ingestion/src/metadata/ingestion/source/tableau.py index ef38637a59b..bfc963b9727 100644 --- a/ingestion/src/metadata/ingestion/source/tableau.py +++ b/ingestion/src/metadata/ingestion/source/tableau.py @@ -14,10 +14,9 @@ Tableau source module import logging import uuid -from typing import Iterable, List, Optional +from typing import Iterable, List import dateutil.parser as dateparser -from pydantic import SecretStr from tableau_api_lib import TableauServerConnection from tableau_api_lib.utils.querying import ( get_views_dataframe, @@ -31,16 +30,22 @@ from metadata.generated.schema.entity.data.dashboard import ( Dashboard as Dashboard_Entity, ) from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.services.connections.dashboard.tableauConnection import ( + TableauConnection, +) from metadata.generated.schema.entity.services.dashboardService import ( DashboardServiceType, ) from metadata.generated.schema.metadataIngestion.workflow import ( OpenMetadataServerConfig, ) +from metadata.generated.schema.metadataIngestion.workflow import ( + Source as WorkflowSource, +) from metadata.generated.schema.type.entityLineage import EntitiesEdge from metadata.generated.schema.type.entityReference import EntityReference -from metadata.ingestion.api.common import ConfigModel, Entity, IncludeFilterPattern -from metadata.ingestion.api.source import Source, SourceStatus +from metadata.ingestion.api.common import Entity +from metadata.ingestion.api.source import InvalidSourceException, Source, SourceStatus from metadata.ingestion.models.table_metadata import Chart, Dashboard, DashboardOwner from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.utils.helpers import get_dashboard_service_or_create @@ -48,25 +53,6 @@ from metadata.utils.helpers import get_dashboard_service_or_create logger = logging.getLogger(__name__) -class TableauSourceConfig(ConfigModel): - """Tableau pydantic source model""" - - username: Optional[str] = None - password: Optional[SecretStr] = None - server: str - api_version: str - env: Optional[str] = "tableau_prod" - site_name: str - site_url: str - db_service_name: Optional[str] = None - service_name: str - service_type: str = DashboardServiceType.Tableau.value - personal_access_token_name: Optional[str] = None - personal_access_token_secret: Optional[str] = None - dashboard_pattern: IncludeFilterPattern = IncludeFilterPattern.allow_all() - chart_pattern: IncludeFilterPattern = IncludeFilterPattern.allow_all() - - class TableauSource(Source[Entity]): """Tableau source entity class @@ -83,25 +69,24 @@ class TableauSource(Source[Entity]): all_dashboard_details: """ - config: TableauSourceConfig + config: WorkflowSource metadata_config: OpenMetadataServerConfig status: SourceStatus def __init__( self, - config: TableauSourceConfig, + config: WorkflowSource, metadata_config: OpenMetadataServerConfig, ): super().__init__() self.config = config self.metadata_config = metadata_config + self.connection_config = self.config.serviceConnection.__root__.config self.client = self.tableau_client() self.service = get_dashboard_service_or_create( - service_name=config.service_name, + service_name=config.serviceName, dashboard_service_type=DashboardServiceType.Tableau.name, - username=config.username, - password=config.password.get_secret_value() if config.password else None, - dashboard_url=config.server, + config=self.config.serviceConnection.__root__.dict(), metadata_config=metadata_config, ) self.status = SourceStatus() @@ -115,31 +100,34 @@ class TableauSource(Source[Entity]): Returns: """ tableau_server_config = { - f"{self.config.env}": { - "server": self.config.server, - "api_version": self.config.api_version, - "site_name": self.config.site_name, - "site_url": self.config.site_url, + f"{self.connection_config.env}": { + "server": self.connection_config.hostPort, + "api_version": self.connection_config.apiVersion, + "site_name": self.connection_config.siteName, + "site_url": self.connection_config.siteName, } } - if self.config.username and self.config.password: - tableau_server_config[self.config.env]["username"] = self.config.username - tableau_server_config[self.config.env][ + if self.connection_config.username and self.connection_config.password: + tableau_server_config[self.connection_config.env][ + "username" + ] = self.connection_config.username + tableau_server_config[self.connection_config.env][ "password" - ] = self.config.password.get_secret_value() + ] = self.connection_config.password.get_secret_value() elif ( - self.config.personal_access_token_name - and self.config.personal_access_token_secret + self.connection_config.personalAccessTokenName + and self.connection_config.personalAccessTokenSecret ): - tableau_server_config[self.config.env][ + tableau_server_config[self.connection_config.env][ "personal_access_token_name" - ] = self.config.personal_access_token_name - tableau_server_config[self.config.env][ + ] = self.connection_config.personalAccessTokenName + tableau_server_config[self.connection_config.env][ "personal_access_token_secret" - ] = self.config.personal_access_token_secret + ] = self.connection_config.personalAccessTokenSecret try: conn = TableauServerConnection( - config_json=tableau_server_config, env=self.config.env + config_json=tableau_server_config, + env=self.connection_config.env, ) conn.sign_in().json() except Exception as err: # pylint: disable=broad-except @@ -148,7 +136,12 @@ class TableauSource(Source[Entity]): @classmethod def create(cls, config_dict: dict, metadata_config: OpenMetadataServerConfig): - config = TableauSourceConfig.parse_obj(config_dict) + config: WorkflowSource = WorkflowSource.parse_obj(config_dict) + connection: TableauConnection = config.serviceConnection.__root__.config + if not isinstance(connection, TableauConnection): + raise InvalidSourceException( + f"Expected TableauConnection, but got {connection}" + ) return cls(config, metadata_config) def prepare(self): @@ -182,8 +175,8 @@ class TableauSource(Source[Entity]): for datasource in datasource_list: try: table_fqdn = datasource.split("(")[1].split(")")[0] - dashboard_fqdn = f"{self.config.service_name}.{dashboard_name}" - table_fqdn = f"{self.config.db_service_name}.{table_fqdn}" + dashboard_fqdn = f"{self.config.serviceName}.{dashboard_name}" + table_fqdn = f"{self.config.serviceName}.{table_fqdn}" table_entity = self.metadata_client.get_by_name( entity=Table, fqdn=table_fqdn ) @@ -240,7 +233,7 @@ class TableauSource(Source[Entity]): service=EntityReference(id=self.service.id, type="dashboardService"), last_modified=dateparser.parse(chart["updatedAt"]).timestamp() * 1000, ) - if self.config.db_service_name: + if self.config.serviceName: yield from self.get_lineage(datasource_list, dashboard_id) def _get_tableau_charts(self): @@ -250,7 +243,7 @@ class TableauSource(Source[Entity]): chart_tags = self.all_dashboard_details["tags"][index] chart_type = self.all_dashboard_details["sheetType"][index] chart_url = ( - f"{self.config.server}/#/site/{self.config.site_name}" + f"{self.connection_config.hostPort}/#/site/{self.connection_config.siteName}" f"{self.all_dashboard_details['contentUrl'][index]}" ) chart_owner = self.all_dashboard_details["owner"][index]