Fix #5416 - Revisit Looker Test Connection (#5422)

Fix #5416 - Revisit Looker Test Connection (#5422)

catalog-rest-service/src/main/resources/json/schema/api/services/ingestionPipelines/testServiceConnection.json

@@ -6,7 +6,7 @@
   "type": "object",
   "properties": {
     "connection": {
-      "description": "Database Connection.",
+      "description": "Connection object.",
       "oneOf": [
         {
           "$ref": "../../../entity/services/databaseService.json#/definitions/databaseConnection"

ingestion/setup.py

@@ -99,7 +99,7 @@ plugins: Dict[str, Set[str]] = {
         "confluent_avro",
     },
     "ldap-users": {"ldap3==2.9.1"},
-    "looker": {"looker-sdk==21.12.2"},
+    "looker": {"looker-sdk>=22.4.0"},
     "mssql": {"sqlalchemy-pytds>=0.3"},
     "pymssql": {"pymssql~=2.2.5"},
     "mssql-odbc": {"pyodbc"},

ingestion/src/metadata/ingestion/source/dashboard/dashboard_source.py

@@ -18,7 +18,7 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.ingestion.api.common import Entity
 from metadata.ingestion.api.source import Source, SourceStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.utils.connections import get_connection
+from metadata.utils.connections import get_connection, test_connection
 from metadata.utils.filters import filter_by_dashboard
 from metadata.utils.logger import ingestion_logger
 
@@ -94,6 +94,8 @@ class DashboardSourceService(Source, ABC):
             self.config.sourceConfig.config
         )
         self.connection = get_connection(self.service_connection)
+        self.test_connection()
+
         self.client = self.connection.client
         self.service = self.metadata.get_service_or_create(
             entity=DashboardService, config=config
@@ -135,7 +137,7 @@ class DashboardSourceService(Source, ABC):
         pass
 
     def test_connection(self) -> None:
-        pass
+        test_connection(self.connection)
 
     def prepare(self):
         pass

ingestion/src/metadata/ingestion/source/dashboard/looker.py

@@ -9,12 +9,9 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import os
 import traceback
 from typing import Any, Iterable, List, Optional
 
-import looker_sdk
-
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
@@ -28,7 +25,6 @@ from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.common import Entity
 from metadata.ingestion.api.source import InvalidSourceException
 from metadata.ingestion.source.dashboard.dashboard_source import DashboardSourceService
 from metadata.utils.filters import filter_by_chart
@@ -48,28 +44,7 @@ class LookerSource(DashboardSourceService):
         metadata_config: OpenMetadataConnection,
     ):
         super().__init__(config, metadata_config)
-        self.client = self.looker_client()
+        self.charts = []
-
-    def check_env(self, env_key):
-        if os.environ.get(env_key):
-            return True
-        return None
-
-    def looker_client(self):
-        try:
-            if not self.check_env("LOOKERSDK_CLIENT_ID"):
-                os.environ["LOOKERSDK_CLIENT_ID"] = self.service_connection.username
-            if not self.check_env("LOOKERSDK_CLIENT_SECRET"):
-                os.environ[
-                    "LOOKERSDK_CLIENT_SECRET"
-                ] = self.service_connection.password.get_secret_value()
-            if not self.check_env("LOOKERSDK_BASE_URL"):
-                os.environ["LOOKERSDK_BASE_URL"] = self.service_connection.hostPort
-            client = looker_sdk.init31()
-            client.me()
-            return client
-        except Exception as err:
-            logger.error(f"ERROR: {repr(err)}")
 
     @classmethod
     def create(cls, config_dict: dict, metadata_config: OpenMetadataConnection):
@@ -115,7 +90,7 @@ class LookerSource(DashboardSourceService):
             displayName=dashboard_details.title,
             description=dashboard_details.description or "",
             charts=get_chart_entities_from_id(
-                chart_ids=self.chart_names,
+                chart_ids=self.charts,
                 metadata=self.metadata,
                 service_name=self.config.serviceName,
             ),
@@ -137,7 +112,6 @@ class LookerSource(DashboardSourceService):
         Metod to fetch charts linked to dashboard
         """
         self.charts = []
-        self.chart_names = []
         for dashboard_elements in dashboard_details.dashboard_elements:
             try:
                 if filter_by_chart(
@@ -148,7 +122,7 @@ class LookerSource(DashboardSourceService):
                     continue
                 om_dashboard_elements = CreateChartRequest(
                     name=dashboard_elements.id,
-                    displayName=dashboard_elements.title or "",
+                    displayName=dashboard_elements.title or dashboard_elements.id,
                     description="",
                     chartType=get_standard_chart_type(dashboard_elements.type).value,
                     chartUrl=f"/dashboard_elements/{dashboard_elements.id}",
@@ -160,8 +134,7 @@ class LookerSource(DashboardSourceService):
                     raise ValueError("Chart(Dashboard Element) without ID")
                 self.status.scanned(dashboard_elements.id)
                 yield om_dashboard_elements
-                self.charts.append(om_dashboard_elements)
+                self.charts.append(dashboard_elements.id)
-                self.chart_names.append(dashboard_elements.id)
             except Exception as err:
                 logger.debug(traceback.format_exc())
                 logger.error(err)

ingestion/src/metadata/ingestion/source/dashboard/metabase.py

@@ -157,6 +157,7 @@ class MetabaseSource(DashboardSourceService):
         Returns:
             Iterable[CreateChartRequest]
         """
+        self.charts = []
         charts = dashboard_details["ordered_cards"]
         for chart in charts:
             try:

ingestion/src/metadata/ingestion/source/dashboard/powerbi.py

@@ -57,6 +57,7 @@ class PowerbiSource(DashboardSourceService):
         metadata_config: OpenMetadataConnection,
     ):
         super().__init__(config, metadata_config)
+        self.charts = []
 
     @classmethod
     def create(cls, config_dict, metadata_config: OpenMetadataConnection):

ingestion/src/metadata/ingestion/source/dashboard/superset.py

@@ -116,6 +116,7 @@ class SupersetSource(DashboardSourceService):
         metadata_config: OpenMetadataConnection,
     ):
         super().__init__(config, metadata_config)
+        self.charts = []
 
     @classmethod
     def create(cls, config_dict: dict, metadata_config: OpenMetadataConnection):

ingestion/src/metadata/ingestion/source/dashboard/tableau.py

@@ -83,6 +83,7 @@ class TableauSource(DashboardSourceService):
     ):
 
         super().__init__(config, metadata_config)
+        self.charts = []
         self.dashboards = get_workbooks_dataframe(self.client).to_dict()
         self.all_dashboard_details = get_views_dataframe(self.client).to_dict()
 

ingestion/src/metadata/ingestion/source/database/common_db_source.py

@@ -12,7 +12,6 @@
 Generic source to build SQL connectors.
 """
 
-import traceback
 from dataclasses import dataclass
 from datetime import datetime
 from typing import Iterable, Optional, Tuple

ingestion/src/metadata/utils/connection_clients.py

@@ -78,3 +78,9 @@ class TableauClient:
 class PowerBiClient:
     def __init__(self, client) -> None:
         self.client = client
+
+
+@dataclass
+class LookerClient:
+    def __init__(self, client) -> None:
+        self.client = client

ingestion/src/metadata/utils/connections.py

@@ -14,6 +14,7 @@ Build and document all supported Engines
 """
 import json
 import logging
+import os
 import traceback
 from functools import singledispatch
 from typing import Union
@@ -28,6 +29,9 @@ from sqlalchemy.orm.session import Session
 from metadata.generated.schema.entity.services.connections.connectionBasicType import (
     ConnectionArguments,
 )
+from metadata.generated.schema.entity.services.connections.dashboard.lookerConnection import (
+    LookerConnection,
+)
 from metadata.generated.schema.entity.services.connections.dashboard.metabaseConnection import (
     MetabaseConnection,
 )
@@ -73,6 +77,7 @@ from metadata.utils.connection_clients import (
     DynamoClient,
     GlueClient,
     KafkaClient,
+    LookerClient,
     MetabaseClient,
     PowerBiClient,
     RedashClient,
@@ -536,3 +541,27 @@ def _(connection: PowerBiClient) -> None:
         raise SourceConnectionException(
             f"Unknown error connecting with {connection} - {err}."
         )
+
+
+@get_connection.register
+def _(connection: LookerConnection, verbose: bool = False):
+    import looker_sdk
+
+    if not os.environ.get("LOOKERSDK_CLIENT_ID"):
+        os.environ["LOOKERSDK_CLIENT_ID"] = connection.username
+    if not os.environ.get("LOOKERSDK_CLIENT_SECRET"):
+        os.environ["LOOKERSDK_CLIENT_SECRET"] = connection.password.get_secret_value()
+    if not os.environ.get("LOOKERSDK_BASE_URL"):
+        os.environ["LOOKERSDK_BASE_URL"] = connection.hostPort
+    client = looker_sdk.init31()
+    return LookerClient(client=client)
+
+
+@test_connection.register
+def _(connection: LookerClient) -> None:
+    try:
+        connection.client.me()
+    except Exception as err:
+        raise SourceConnectionException(
+            f"Unknown error connecting with {connection} - {err}."
+        )