From 0f8ebf4afea4402162b81cfc7e95a821cb75fa4a Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Wed, 26 Jul 2023 16:07:48 +0530 Subject: [PATCH] Fixed Tableau-Bigquery lineage issue (#12570) --- .../source/dashboard/tableau/client.py | 21 +++++++++++++++++++ .../source/dashboard/tableau/connection.py | 1 + .../source/dashboard/tableau/metadata.py | 5 +++++ ingestion/src/metadata/utils/helpers.py | 2 +- .../testConnections/dashboard/tableau.json | 6 ++++++ .../database/bigQueryConnection.json | 4 ++++ 6 files changed, 38 insertions(+), 1 deletion(-) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau/client.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau/client.py index f96c6803f93..b564a2079c0 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau/client.py @@ -45,6 +45,12 @@ class TableauOwnersNotFound(Exception): """ +class TableauDataModelsException(Exception): + """ + Raise when Data Source information is not retrieved from the Tableau Graphql Query + """ + + class TableauClient: """ Wrapper to TableauServerConnection @@ -116,6 +122,21 @@ class TableauClient: ) ] + def test_get_datamodels(self): + """ + Method to test the datamodels + """ + data = self._query_datasources(entities_per_page=1, offset=0) + if data: + return data + raise TableauDataModelsException( + "Unable to fetch Data Sources from tableau\n" + "Please check if the Tableau Metadata APIs are enabled for you Tableau instance\n" + "For more information on enabling the Tableau Metadata APIs follow the link below\n" + "https://help.tableau.com/current/api/metadata_api/en-us/docs/meta_api_start.html" + "#enable-the-tableau-metadata-api-for-tableau-server\n" + ) + def _query_datasources( self, entities_per_page: int, offset: int ) -> Optional[TableauDatasources]: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau/connection.py index 6002903947b..8f52f6121a8 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau/connection.py @@ -89,6 +89,7 @@ def test_connection( parameter_dict=TABLEAU_GET_VIEWS_PARAM_DICT, ), "GetOwners": client.get_owners, + "GetDataModels": client.test_get_datamodels, } test_connection_steps( diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py index 9edbc342944..0912e62563e 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py @@ -29,6 +29,9 @@ from metadata.generated.schema.entity.data.table import Column, DataType, Table from metadata.generated.schema.entity.services.connections.dashboard.tableauConnection import ( TableauConnection, ) +from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import ( + BigQueryConnection, +) from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) @@ -367,6 +370,8 @@ class TableauSource(DashboardServiceSource): if table.database and table.database.name else database_schema_table.get("database") ) + if isinstance(db_service_entity.connection.config, BigQueryConnection): + database_name = None database_name = get_database_name_for_lineage( db_service_entity, database_name ) diff --git a/ingestion/src/metadata/utils/helpers.py b/ingestion/src/metadata/utils/helpers.py index a74bdfe3bfd..4485a9956da 100644 --- a/ingestion/src/metadata/utils/helpers.py +++ b/ingestion/src/metadata/utils/helpers.py @@ -440,7 +440,7 @@ def is_safe_sql_query(sql_query: str) -> bool: def get_database_name_for_lineage( - db_service_entity: DatabaseService, default_db_name: str + db_service_entity: DatabaseService, default_db_name: Optional[str] ) -> Optional[str]: # If the database service supports multiple db or # database service connection details are not available diff --git a/openmetadata-service/src/main/resources/json/data/testConnections/dashboard/tableau.json b/openmetadata-service/src/main/resources/json/data/testConnections/dashboard/tableau.json index 8691507d090..5168e891e49 100644 --- a/openmetadata-service/src/main/resources/json/data/testConnections/dashboard/tableau.json +++ b/openmetadata-service/src/main/resources/json/data/testConnections/dashboard/tableau.json @@ -27,6 +27,12 @@ "description": "Validate if the Owner information is retrieved for Workbooks", "errorMessage": "Failed to fetch Workbook Owners, please validate if user has access to fetch Owners", "mandatory": false + }, + { + "name": "GetDataModels", + "description": "Validate if the Data Sources(Data Models) information is retrieved for Workbooks", + "errorMessage": "Failed to fetch Workbook Data Sources, please validate if user has access to fetch Data Sources and Metadata API is enabled for tableau server", + "mandatory": false } ] } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json index 5c8020e8aa4..d4872b75c8f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json @@ -93,6 +93,10 @@ "title": "Supports Profiler", "$ref": "../connectionBasicType.json#/definitions/supportsProfiler" }, + "supportsDatabase": { + "title": "Supports Database", + "$ref": "../connectionBasicType.json#/definitions/supportsDatabase" + }, "supportsQueryComment": { "title": "Supports Query Comment", "$ref": "../connectionBasicType.json#/definitions/supportsQueryComment"