yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-02 05:33:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #5416 - Revisit Looker Test Connection (#5422)

Browse Source 
Fix #5416 - Revisit Looker Test Connection (#5422)
This commit is contained in:
Pere Miquel Brull 2022-06-13 08:07:04 +02:00 committed by GitHub
parent 51a275c481
commit d9d30b2cba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 49 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
catalog-rest-service/src/main/resources/json/schema/api/services/ingestionPipelines/testServiceConnection.json
View File

@ -6,7 +6,7 @@
"type": "object",
"properties": {
"connection": {
"description": "Database Connection.",
"description": "Connection object.",
"oneOf": [
{
"$ref": "../../../entity/services/databaseService.json#/definitions/databaseConnection"

2
ingestion/setup.py
View File

@ -99,7 +99,7 @@ plugins: Dict[str, Set[str]] = {
"confluent_avro",
},
"ldap-users": {"ldap3==2.9.1"},
"looker": {"looker-sdk==21.12.2"},
"looker": {"looker-sdk>=22.4.0"},
"mssql": {"sqlalchemy-pytds>=0.3"},
"pymssql": {"pymssql~=2.2.5"},
"mssql-odbc": {"pyodbc"},

6
ingestion/src/metadata/ingestion/source/dashboard/dashboard_source.py
View File

@ -18,7 +18,7 @@ from metadata.generated.schema.metadataIngestion.workflow import (
from metadata.ingestion.api.common import Entity
from metadata.ingestion.api.source import Source, SourceStatus
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.utils.connections import get_connection
from metadata.utils.connections import get_connection, test_connection
from metadata.utils.filters import filter_by_dashboard
from metadata.utils.logger import ingestion_logger
@ -94,6 +94,8 @@ class DashboardSourceService(Source, ABC):
self.config.sourceConfig.config
)
self.connection = get_connection(self.service_connection)
self.test_connection()
self.client = self.connection.client
self.service = self.metadata.get_service_or_create(
entity=DashboardService, config=config
@ -135,7 +137,7 @@ class DashboardSourceService(Source, ABC):
pass
def test_connection(self) -> None:
pass
test_connection(self.connection)
def prepare(self):
pass

35
ingestion/src/metadata/ingestion/source/dashboard/looker.py
View File

@ -9,12 +9,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import traceback
from typing import Any, Iterable, List, Optional
import looker_sdk
from metadata.generated.schema.api.data.createChart import CreateChartRequest
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
@ -28,7 +25,6 @@ from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.api.common import Entity
from metadata.ingestion.api.source import InvalidSourceException
from metadata.ingestion.source.dashboard.dashboard_source import DashboardSourceService
from metadata.utils.filters import filter_by_chart
@ -48,28 +44,7 @@ class LookerSource(DashboardSourceService):
metadata_config: OpenMetadataConnection,
):
super().__init__(config, metadata_config)
self.client = self.looker_client()
def check_env(self, env_key):
if os.environ.get(env_key):
return True
return None
def looker_client(self):
try:
if not self.check_env("LOOKERSDK_CLIENT_ID"):
os.environ["LOOKERSDK_CLIENT_ID"] = self.service_connection.username
if not self.check_env("LOOKERSDK_CLIENT_SECRET"):
os.environ[
"LOOKERSDK_CLIENT_SECRET"
] = self.service_connection.password.get_secret_value()
if not self.check_env("LOOKERSDK_BASE_URL"):
os.environ["LOOKERSDK_BASE_URL"] = self.service_connection.hostPort
client = looker_sdk.init31()
client.me()
return client
except Exception as err:
logger.error(f"ERROR: {repr(err)}")
self.charts = []
@classmethod
def create(cls, config_dict: dict, metadata_config: OpenMetadataConnection):
@ -115,7 +90,7 @@ class LookerSource(DashboardSourceService):
displayName=dashboard_details.title,
description=dashboard_details.description or "",
charts=get_chart_entities_from_id(
chart_ids=self.chart_names,
chart_ids=self.charts,
metadata=self.metadata,
service_name=self.config.serviceName,
),
@ -137,7 +112,6 @@ class LookerSource(DashboardSourceService):
Metod to fetch charts linked to dashboard
"""
self.charts = []
self.chart_names = []
for dashboard_elements in dashboard_details.dashboard_elements:
try:
if filter_by_chart(
@ -148,7 +122,7 @@ class LookerSource(DashboardSourceService):
continue
om_dashboard_elements = CreateChartRequest(
name=dashboard_elements.id,
displayName=dashboard_elements.title or "",
displayName=dashboard_elements.title or dashboard_elements.id,
description="",
chartType=get_standard_chart_type(dashboard_elements.type).value,
chartUrl=f"/dashboard_elements/{dashboard_elements.id}",
@ -160,8 +134,7 @@ class LookerSource(DashboardSourceService):
raise ValueError("Chart(Dashboard Element) without ID")
self.status.scanned(dashboard_elements.id)
yield om_dashboard_elements
self.charts.append(om_dashboard_elements)
self.chart_names.append(dashboard_elements.id)
self.charts.append(dashboard_elements.id)
except Exception as err:
logger.debug(traceback.format_exc())
logger.error(err)

1
ingestion/src/metadata/ingestion/source/dashboard/metabase.py
View File

@ -157,6 +157,7 @@ class MetabaseSource(DashboardSourceService):
Returns:
Iterable[CreateChartRequest]
"""
self.charts = []
charts = dashboard_details["ordered_cards"]
for chart in charts:
try:

1
ingestion/src/metadata/ingestion/source/dashboard/powerbi.py
View File

@ -57,6 +57,7 @@ class PowerbiSource(DashboardSourceService):
metadata_config: OpenMetadataConnection,
):
super().__init__(config, metadata_config)
self.charts = []
@classmethod
def create(cls, config_dict, metadata_config: OpenMetadataConnection):

1
ingestion/src/metadata/ingestion/source/dashboard/superset.py
View File

@ -116,6 +116,7 @@ class SupersetSource(DashboardSourceService):
metadata_config: OpenMetadataConnection,
):
super().__init__(config, metadata_config)
self.charts = []
@classmethod
def create(cls, config_dict: dict, metadata_config: OpenMetadataConnection):

1
ingestion/src/metadata/ingestion/source/dashboard/tableau.py
View File

@ -83,6 +83,7 @@ class TableauSource(DashboardSourceService):
):
super().__init__(config, metadata_config)
self.charts = []
self.dashboards = get_workbooks_dataframe(self.client).to_dict()
self.all_dashboard_details = get_views_dataframe(self.client).to_dict()

1
ingestion/src/metadata/ingestion/source/database/common_db_source.py
View File

@ -12,7 +12,6 @@
Generic source to build SQL connectors.
"""
import traceback
from dataclasses import dataclass
from datetime import datetime
from typing import Iterable, Optional, Tuple

6
ingestion/src/metadata/utils/connection_clients.py
View File

@ -78,3 +78,9 @@ class TableauClient:
class PowerBiClient:
def __init__(self, client) -> None:
self.client = client
@dataclass
class LookerClient:
def __init__(self, client) -> None:
self.client = client

29
ingestion/src/metadata/utils/connections.py
View File

@ -14,6 +14,7 @@ Build and document all supported Engines
"""
import json
import logging
import os
import traceback
from functools import singledispatch
from typing import Union
@ -28,6 +29,9 @@ from sqlalchemy.orm.session import Session
from metadata.generated.schema.entity.services.connections.connectionBasicType import (
ConnectionArguments,
)
from metadata.generated.schema.entity.services.connections.dashboard.lookerConnection import (
LookerConnection,
)
from metadata.generated.schema.entity.services.connections.dashboard.metabaseConnection import (
MetabaseConnection,
)
@ -73,6 +77,7 @@ from metadata.utils.connection_clients import (
DynamoClient,
GlueClient,
KafkaClient,
LookerClient,
MetabaseClient,
PowerBiClient,
RedashClient,
@ -536,3 +541,27 @@ def _(connection: PowerBiClient) -> None:
raise SourceConnectionException(
f"Unknown error connecting with {connection} - {err}."
)
@get_connection.register
def _(connection: LookerConnection, verbose: bool = False):
import looker_sdk
if not os.environ.get("LOOKERSDK_CLIENT_ID"):
os.environ["LOOKERSDK_CLIENT_ID"] = connection.username
if not os.environ.get("LOOKERSDK_CLIENT_SECRET"):
os.environ["LOOKERSDK_CLIENT_SECRET"] = connection.password.get_secret_value()
if not os.environ.get("LOOKERSDK_BASE_URL"):
os.environ["LOOKERSDK_BASE_URL"] = connection.hostPort
client = looker_sdk.init31()
return LookerClient(client=client)
@test_connection.register
def _(connection: LookerClient) -> None:
try:
connection.client.me()
except Exception as err:
raise SourceConnectionException(
f"Unknown error connecting with {connection} - {err}."
)