From d9d30b2cba2fd303fb76e527f39190d636ed7eb2 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 13 Jun 2022 08:07:04 +0200 Subject: [PATCH] Fix #5416 - Revisit Looker Test Connection (#5422) Fix #5416 - Revisit Looker Test Connection (#5422) --- .../testServiceConnection.json | 2 +- ingestion/setup.py | 2 +- .../source/dashboard/dashboard_source.py | 6 ++-- .../ingestion/source/dashboard/looker.py | 35 +++---------------- .../ingestion/source/dashboard/metabase.py | 1 + .../ingestion/source/dashboard/powerbi.py | 1 + .../ingestion/source/dashboard/superset.py | 1 + .../ingestion/source/dashboard/tableau.py | 1 + .../source/database/common_db_source.py | 1 - .../src/metadata/utils/connection_clients.py | 6 ++++ ingestion/src/metadata/utils/connections.py | 29 +++++++++++++++ 11 files changed, 49 insertions(+), 36 deletions(-) diff --git a/catalog-rest-service/src/main/resources/json/schema/api/services/ingestionPipelines/testServiceConnection.json b/catalog-rest-service/src/main/resources/json/schema/api/services/ingestionPipelines/testServiceConnection.json index 790bcfaf36f..3e5a0274089 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/services/ingestionPipelines/testServiceConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/services/ingestionPipelines/testServiceConnection.json @@ -6,7 +6,7 @@ "type": "object", "properties": { "connection": { - "description": "Database Connection.", + "description": "Connection object.", "oneOf": [ { "$ref": "../../../entity/services/databaseService.json#/definitions/databaseConnection" diff --git a/ingestion/setup.py b/ingestion/setup.py index 243c7c12311..2bd5e44e317 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -99,7 +99,7 @@ plugins: Dict[str, Set[str]] = { "confluent_avro", }, "ldap-users": {"ldap3==2.9.1"}, - "looker": {"looker-sdk==21.12.2"}, + "looker": {"looker-sdk>=22.4.0"}, "mssql": {"sqlalchemy-pytds>=0.3"}, "pymssql": {"pymssql~=2.2.5"}, "mssql-odbc": {"pyodbc"}, diff --git a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_source.py b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_source.py index 8c577e4e983..1674170d6cf 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_source.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_source.py @@ -18,7 +18,7 @@ from metadata.generated.schema.metadataIngestion.workflow import ( from metadata.ingestion.api.common import Entity from metadata.ingestion.api.source import Source, SourceStatus from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.utils.connections import get_connection +from metadata.utils.connections import get_connection, test_connection from metadata.utils.filters import filter_by_dashboard from metadata.utils.logger import ingestion_logger @@ -94,6 +94,8 @@ class DashboardSourceService(Source, ABC): self.config.sourceConfig.config ) self.connection = get_connection(self.service_connection) + self.test_connection() + self.client = self.connection.client self.service = self.metadata.get_service_or_create( entity=DashboardService, config=config @@ -135,7 +137,7 @@ class DashboardSourceService(Source, ABC): pass def test_connection(self) -> None: - pass + test_connection(self.connection) def prepare(self): pass diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker.py b/ingestion/src/metadata/ingestion/source/dashboard/looker.py index 2ba7f4e1e6a..ed716ac8453 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker.py @@ -9,12 +9,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import traceback from typing import Any, Iterable, List, Optional -import looker_sdk - from metadata.generated.schema.api.data.createChart import CreateChartRequest from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest @@ -28,7 +25,6 @@ from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) from metadata.generated.schema.type.entityReference import EntityReference -from metadata.ingestion.api.common import Entity from metadata.ingestion.api.source import InvalidSourceException from metadata.ingestion.source.dashboard.dashboard_source import DashboardSourceService from metadata.utils.filters import filter_by_chart @@ -48,28 +44,7 @@ class LookerSource(DashboardSourceService): metadata_config: OpenMetadataConnection, ): super().__init__(config, metadata_config) - self.client = self.looker_client() - - def check_env(self, env_key): - if os.environ.get(env_key): - return True - return None - - def looker_client(self): - try: - if not self.check_env("LOOKERSDK_CLIENT_ID"): - os.environ["LOOKERSDK_CLIENT_ID"] = self.service_connection.username - if not self.check_env("LOOKERSDK_CLIENT_SECRET"): - os.environ[ - "LOOKERSDK_CLIENT_SECRET" - ] = self.service_connection.password.get_secret_value() - if not self.check_env("LOOKERSDK_BASE_URL"): - os.environ["LOOKERSDK_BASE_URL"] = self.service_connection.hostPort - client = looker_sdk.init31() - client.me() - return client - except Exception as err: - logger.error(f"ERROR: {repr(err)}") + self.charts = [] @classmethod def create(cls, config_dict: dict, metadata_config: OpenMetadataConnection): @@ -115,7 +90,7 @@ class LookerSource(DashboardSourceService): displayName=dashboard_details.title, description=dashboard_details.description or "", charts=get_chart_entities_from_id( - chart_ids=self.chart_names, + chart_ids=self.charts, metadata=self.metadata, service_name=self.config.serviceName, ), @@ -137,7 +112,6 @@ class LookerSource(DashboardSourceService): Metod to fetch charts linked to dashboard """ self.charts = [] - self.chart_names = [] for dashboard_elements in dashboard_details.dashboard_elements: try: if filter_by_chart( @@ -148,7 +122,7 @@ class LookerSource(DashboardSourceService): continue om_dashboard_elements = CreateChartRequest( name=dashboard_elements.id, - displayName=dashboard_elements.title or "", + displayName=dashboard_elements.title or dashboard_elements.id, description="", chartType=get_standard_chart_type(dashboard_elements.type).value, chartUrl=f"/dashboard_elements/{dashboard_elements.id}", @@ -160,8 +134,7 @@ class LookerSource(DashboardSourceService): raise ValueError("Chart(Dashboard Element) without ID") self.status.scanned(dashboard_elements.id) yield om_dashboard_elements - self.charts.append(om_dashboard_elements) - self.chart_names.append(dashboard_elements.id) + self.charts.append(dashboard_elements.id) except Exception as err: logger.debug(traceback.format_exc()) logger.error(err) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase.py index 7a2bfa042c6..dbf5273237c 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase.py @@ -157,6 +157,7 @@ class MetabaseSource(DashboardSourceService): Returns: Iterable[CreateChartRequest] """ + self.charts = [] charts = dashboard_details["ordered_cards"] for chart in charts: try: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi.py index c8ffad5e958..332ffb2ef47 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi.py @@ -57,6 +57,7 @@ class PowerbiSource(DashboardSourceService): metadata_config: OpenMetadataConnection, ): super().__init__(config, metadata_config) + self.charts = [] @classmethod def create(cls, config_dict, metadata_config: OpenMetadataConnection): diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset.py b/ingestion/src/metadata/ingestion/source/dashboard/superset.py index c4c74be843e..a4953cebebc 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset.py @@ -116,6 +116,7 @@ class SupersetSource(DashboardSourceService): metadata_config: OpenMetadataConnection, ): super().__init__(config, metadata_config) + self.charts = [] @classmethod def create(cls, config_dict: dict, metadata_config: OpenMetadataConnection): diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau.py index 036727351a4..228e754da86 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau.py @@ -83,6 +83,7 @@ class TableauSource(DashboardSourceService): ): super().__init__(config, metadata_config) + self.charts = [] self.dashboards = get_workbooks_dataframe(self.client).to_dict() self.all_dashboard_details = get_views_dataframe(self.client).to_dict() diff --git a/ingestion/src/metadata/ingestion/source/database/common_db_source.py b/ingestion/src/metadata/ingestion/source/database/common_db_source.py index 6443a3f1c0e..72c3792b76a 100644 --- a/ingestion/src/metadata/ingestion/source/database/common_db_source.py +++ b/ingestion/src/metadata/ingestion/source/database/common_db_source.py @@ -12,7 +12,6 @@ Generic source to build SQL connectors. """ -import traceback from dataclasses import dataclass from datetime import datetime from typing import Iterable, Optional, Tuple diff --git a/ingestion/src/metadata/utils/connection_clients.py b/ingestion/src/metadata/utils/connection_clients.py index 2c6d6e914b4..8a9063abb93 100644 --- a/ingestion/src/metadata/utils/connection_clients.py +++ b/ingestion/src/metadata/utils/connection_clients.py @@ -78,3 +78,9 @@ class TableauClient: class PowerBiClient: def __init__(self, client) -> None: self.client = client + + +@dataclass +class LookerClient: + def __init__(self, client) -> None: + self.client = client diff --git a/ingestion/src/metadata/utils/connections.py b/ingestion/src/metadata/utils/connections.py index 1866116d547..08b8f23865f 100644 --- a/ingestion/src/metadata/utils/connections.py +++ b/ingestion/src/metadata/utils/connections.py @@ -14,6 +14,7 @@ Build and document all supported Engines """ import json import logging +import os import traceback from functools import singledispatch from typing import Union @@ -28,6 +29,9 @@ from sqlalchemy.orm.session import Session from metadata.generated.schema.entity.services.connections.connectionBasicType import ( ConnectionArguments, ) +from metadata.generated.schema.entity.services.connections.dashboard.lookerConnection import ( + LookerConnection, +) from metadata.generated.schema.entity.services.connections.dashboard.metabaseConnection import ( MetabaseConnection, ) @@ -73,6 +77,7 @@ from metadata.utils.connection_clients import ( DynamoClient, GlueClient, KafkaClient, + LookerClient, MetabaseClient, PowerBiClient, RedashClient, @@ -536,3 +541,27 @@ def _(connection: PowerBiClient) -> None: raise SourceConnectionException( f"Unknown error connecting with {connection} - {err}." ) + + +@get_connection.register +def _(connection: LookerConnection, verbose: bool = False): + import looker_sdk + + if not os.environ.get("LOOKERSDK_CLIENT_ID"): + os.environ["LOOKERSDK_CLIENT_ID"] = connection.username + if not os.environ.get("LOOKERSDK_CLIENT_SECRET"): + os.environ["LOOKERSDK_CLIENT_SECRET"] = connection.password.get_secret_value() + if not os.environ.get("LOOKERSDK_BASE_URL"): + os.environ["LOOKERSDK_BASE_URL"] = connection.hostPort + client = looker_sdk.init31() + return LookerClient(client=client) + + +@test_connection.register +def _(connection: LookerClient) -> None: + try: + connection.client.me() + except Exception as err: + raise SourceConnectionException( + f"Unknown error connecting with {connection} - {err}." + )