draft: implementation of lightdash connector. (#12957)

* Fixes a bug while patching the description of a TestCase

* Update docker-compose.yml

* Update docker-compose.yml

* Ran pre-commit checks and linter

* Added some clarifying points and fixed some grammatical errors in the documentation for installation instructions.

* revert changes made to docs

* implementation of lightdash connector.

* ui: add icon for lightdash service

* chore: update lightdash image

* chore: update the icon

* Cleaned up code (took out debug statements, etc.). Still TODO: yield_dashboard_lineage_details() not being called just yet.

* fix: ran linting

* added null checks

* Delete openmetadata-server.md

---------

Co-authored-by: Sachin Chaurasiya <sachinchaurasiyachotey87@gmail.com>
Co-authored-by: Teddy <teddy.crepineau@gmail.com>
This commit is contained in:
gauthk6 2023-08-31 08:28:07 -07:00 committed by GitHub
parent ac28d85973
commit ba2201f4ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 571 additions and 2 deletions

View File

@ -0,0 +1,24 @@
source:
type: lightdash
serviceName: local_test2
serviceConnection:
config:
type: Lightdash
hostPort: https://app.lightdash.cloud
apiKey: <apiKey>
projectUUID: <projectUUID>
spaceUUID: <spaceUUID>
sourceConfig:
config:
type: DashboardMetadata
dashboardFilterPattern: {}
chartFilterPattern: {}
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://localhost:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

View File

@ -0,0 +1,140 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
REST Auth & Client for Lightdash
"""
import traceback
from typing import List
from metadata.ingestion.ometa.client import REST, ClientConfig
from metadata.ingestion.source.dashboard.lightdash.models import (
LightdashChart,
LightdashDashboard,
)
from metadata.utils.logger import utils_logger
logger = utils_logger()
class LightdashApiClient:
"""
REST Auth & Client for Lightdash
"""
client: REST
def __init__(self, config):
self.config = config
client_config = ClientConfig(
base_url=self.config.hostPort,
api_version="",
access_token=self.config.apiKey.get_secret_value(),
auth_header="Authorization",
auth_token_mode="ApiKey",
allow_redirects=True,
)
self.client = REST(client_config)
def get_org(self):
"""GET api/org"""
return self.client.get(
"/api/v1/org",
)
def get_charts_list(self) -> List[LightdashChart]:
"""
Get List of all charts
"""
try:
response = self.client.get(
f"api/v1/projects/{self.config.projectUUID}/charts"
)
response_json_results = response.get("results")
if response_json_results is None:
logger.warning(
"Failed to fetch the charts list for the Lightdash Connector"
)
return []
if len(response_json_results) > 0:
charts_list = []
for chart in response_json_results:
charts_list.append(LightdashChart(**chart))
return charts_list
except Exception:
logger.debug(traceback.format_exc())
logger.warning(
"Failed to fetch the charts list for the Lightdash Connector"
)
return []
def get_dashboards_list(self) -> List[LightdashDashboard]:
"""
Get List of all charts
"""
try:
response = self.client.get(
f"api/v1/projects/{self.config.projectUUID}/spaces/{self.config.spaceUUID}"
)
results = response.get("results")
if results is None:
logger.warning(
"Failed to fetch the dashboard list for the Lightdash Connector"
)
return []
dashboards_raw = results["dashboards"]
if len(dashboards_raw) > 0:
dashboards_list = []
for dashboard in dashboards_raw:
dashboards_list.append(LightdashDashboard(**dashboard))
self.add_dashboard_lineage(dashboards_list=dashboards_list)
return dashboards_list
except Exception:
logger.debug(traceback.format_exc())
logger.warning(
"Failed to fetch the dashboard list for the Lightdash Connector"
)
return []
def add_dashboard_lineage(self, dashboards_list) -> None:
charts_uuid_list = []
for dashboard in dashboards_list:
response = self.client.get(f"api/v1/dashboards/{dashboard.uuid}")
response_json_results = response.get("results")
if response_json_results is None:
logger.warning(
"Failed to fetch dashboard charts for the Lightdash Connector"
)
return
charts = response_json_results["tiles"]
charts_properties = [chart["properties"] for chart in charts]
for chart in charts_properties:
charts_uuid_list.append(chart["savedChartUuid"])
dashboard.charts = self.get_charts_objects(charts_uuid_list)
def get_charts_objects(self, charts_uuid_list) -> List[LightdashChart]:
all_charts = self.get_charts_list()
charts_objects = []
for chart_uuid in charts_uuid_list:
for chart in all_charts:
if chart.uuid == chart_uuid:
charts_objects.append(chart)
return charts_objects

View File

@ -0,0 +1,68 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Source connection handler
"""
from typing import Optional
from metadata.generated.schema.entity.automations.workflow import (
Workflow as AutomationWorkflow,
)
from metadata.generated.schema.entity.services.connections.dashboard.lightdashConnection import (
LightdashConnection,
)
from metadata.ingestion.connections.test_connections import (
SourceConnectionException,
test_connection_steps,
)
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.dashboard.lightdash.client import LightdashApiClient
from metadata.utils.logger import ingestion_logger
logger = ingestion_logger()
def get_connection(connection: LightdashConnection) -> LightdashApiClient:
"""
Create connection
"""
try:
logger.debug("creating a new Lightdash connection")
return LightdashApiClient(connection)
except Exception as exc:
msg = "Unknown error connecting with {connection}: {exc}."
raise SourceConnectionException(msg) from exc
def test_connection(
metadata: OpenMetadata,
client: LightdashApiClient,
service_connection: LightdashConnection,
automation_workflow: Optional[AutomationWorkflow] = None,
) -> None:
"""
Test connection. This can be executed either as part
of a metadata workflow or during an Automation Workflow
"""
def custom_executor():
return client.get_dashboards_list()
test_fn = {"GetDashboards": custom_executor}
test_connection_steps(
metadata=metadata,
test_fn=test_fn,
service_type=service_connection.type.value,
automation_workflow=automation_workflow,
)

View File

@ -0,0 +1,169 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Lightdash source module"""
import traceback
from typing import Iterable, List, Optional
from metadata.generated.schema.api.data.createChart import CreateChartRequest
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
from metadata.generated.schema.entity.data.chart import Chart
from metadata.generated.schema.entity.services.connections.dashboard.lightdashConnection import (
LightdashConnection,
)
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection,
)
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.ingestion.api.source import InvalidSourceException
from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
from metadata.ingestion.source.dashboard.lightdash.models import (
LightdashChart,
LightdashDashboard,
)
from metadata.utils import fqn
from metadata.utils.filters import filter_by_chart
from metadata.utils.helpers import clean_uri, replace_special_with
from metadata.utils.logger import ingestion_logger
logger = ingestion_logger()
class LightdashSource(DashboardServiceSource):
"""
Lightdash Source Class
"""
config: WorkflowSource
metadata_config: OpenMetadataConnection
@classmethod
def create(cls, config_dict, metadata_config: OpenMetadataConnection):
config = WorkflowSource.parse_obj(config_dict)
connection: LightdashConnection = config.serviceConnection.__root__.config
if not isinstance(connection, LightdashConnection):
raise InvalidSourceException(
f"Expected LightdashConnection, but got {connection}"
)
return cls(config, metadata_config)
def __init__(
self,
config: WorkflowSource,
metadata_config: OpenMetadataConnection,
):
super().__init__(config, metadata_config)
self.charts: List[LightdashChart] = []
def prepare(self):
self.charts = self.client.get_charts_list()
return super().prepare()
def get_dashboards_list(self) -> Optional[List[LightdashDashboard]]:
"""
Get List of all dashboards
"""
return self.client.get_dashboards_list()
def get_dashboard_name(self, dashboard: LightdashDashboard) -> str:
"""
Get Dashboard Name
"""
return dashboard.name
def get_dashboard_details(
self, dashboard: LightdashDashboard
) -> LightdashDashboard:
"""
Get Dashboard Details
"""
return dashboard
def yield_dashboard(
self, dashboard_details: LightdashDashboard
) -> Iterable[CreateDashboardRequest]:
"""
Method to Get Dashboard Entity
"""
try:
dashboard_url = (
f"{clean_uri(self.service_connection.hostPort)}/dashboard/{dashboard_details.uuid}-"
f"{replace_special_with(raw=dashboard_details.name.lower(), replacement='-')}"
)
dashboard_request = CreateDashboardRequest(
name=dashboard_details.uuid,
sourceUrl=dashboard_url,
displayName=dashboard_details.name,
description=dashboard_details.description,
charts=[
fqn.build(
self.metadata,
entity_type=Chart,
service_name=self.context.dashboard_service.fullyQualifiedName.__root__,
chart_name=chart.name.__root__,
)
for chart in self.context.charts
],
service=self.context.dashboard_service.fullyQualifiedName.__root__,
)
yield dashboard_request
self.register_record(dashboard_request=dashboard_request)
except Exception as exc: # pylint: disable=broad-except
logger.debug(traceback.format_exc())
logger.warning(
f"Error creating dashboard [{dashboard_details.name}]: {exc}"
)
def yield_dashboard_chart(
self, dashboard_details: LightdashChart
) -> Optional[Iterable[CreateChartRequest]]:
"""Get chart method
Args:
dashboard_details:
Returns:
Iterable[CreateChartRequest]
"""
charts = self.charts
for chart in charts:
try:
chart_url = (
f"{clean_uri(self.service_connection.hostPort)}/question/{chart.uuid}-"
f"{replace_special_with(raw=chart.name.lower(), replacement='-')}"
)
if filter_by_chart(self.source_config.chartFilterPattern, chart.name):
self.status.filter(chart.name, "Chart Pattern not allowed")
continue
yield CreateChartRequest(
name=chart.uuid,
displayName=chart.name,
description=chart.description,
sourceUrl=chart_url,
service=self.context.dashboard_service.fullyQualifiedName.__root__,
)
self.status.scanned(chart.name)
except Exception as exc: # pylint: disable=broad-except
logger.debug(traceback.format_exc())
logger.warning(f"Error creating chart [{chart}]: {exc}")
def yield_dashboard_lineage_details(
self,
dashboard_details: LightdashDashboard,
db_service_name: Optional[str],
) -> Optional[Iterable[AddLineageRequest]]:
"""Get lineage method
Args:
dashboard_details
"""

View File

@ -0,0 +1,46 @@
"""Lightdash models"""
from typing import List, Optional
from pydantic import BaseModel
class LightdashChart(BaseModel):
"""
Lightdash chart model
"""
name: str
organizationUuid: str
uuid: str
description: Optional[str]
projectUuid: str
spaceUuid: str
pinnedListUuid: Optional[str]
spaceName: str
chartType: Optional[str]
dashboardUuid: Optional[str]
dashboardName: Optional[str]
class LightdashDashboard(BaseModel):
organizationUuid: str
name: str
description: Optional[str]
uuid: str
projectUuid: str
updatedAt: str
spaceUuid: str
views: float
firstViewedAt: str
pinnedListUuid: Optional[str]
pinnedListOrder: Optional[float]
charts: Optional[List[LightdashChart]]
class LightdashChartList(BaseModel):
charts: Optional[List[LightdashChart]]
class LightdashDashboardList(BaseModel):
dashboards: Optional[List[LightdashDashboard]]

View File

@ -0,0 +1,25 @@
source:
type: redshift
serviceName: local_redshift
serviceConnection:
config:
hostPort: $E2E_REDSHIFT_HOST_PORT
username: $E2E_REDSHIFT_USERNAME
password: $E2E_REDSHIFT_PASSWORD
database: $E2E_REDSHIFT_DATABASE
type: Redshift
sourceConfig:
config:
schemaFilterPattern:
includes:
- dbt_jaffle
sink:
type: metadata-rest
config: {}
workflowConfig:
loggerLevel: DEBUG
openMetadataServerConfig:
hostPort: http://localhost:8585/api
authProvider: openmetadata
securityConfig:
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

View File

@ -0,0 +1,15 @@
{
"name": "Lightdash",
"displayName": "Lightdash Test Connection",
"description": "This Test Connection validates the access against the server and basic metadata extraction of dashboards and charts.",
"steps": [
{
"name": "GetDashboards",
"description": "List all the dashboards available to the user",
"errorMessage": "Failed to fetch orgs, please validate the credentials or validate if user has access to fetch orgs",
"shortCircuit": true,
"mandatory": true
}
]
}

View File

@ -0,0 +1,60 @@
{
"$id": "https://open-metadata.org/schema/entity/services/connections/dashboard/lightdashConnection.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "LightdashConnection",
"description": "Lightdash Connection Config",
"type": "object",
"javaType": "org.openmetadata.schema.services.connections.dashboard.LightdashConnection",
"definitions": {
"lightdashType": {
"description": "Lightdash service type",
"type": "string",
"enum": ["Lightdash"],
"default": "Lightdash"
}
},
"properties": {
"type": {
"title": "Service Type",
"description": "Service Type",
"$ref": "#/definitions/lightdashType",
"default": "Lightdash"
},
"hostPort": {
"expose": true,
"title": "Host Port",
"description": "Address for your running Lightdash instance",
"type": "string",
"format": "uri",
"default": "http://localhost:5000"
},
"apiKey": {
"title": "API Key",
"description": "The personal access token you can generate in the Lightdash app under the user settings",
"type": "string",
"format": "password"
},
"projectUUID": {
"title": "Project UUID",
"description": "The Project UUID for your Lightdash instance",
"type": "string"
},
"spaceUUID": {
"title": "Space UUID",
"description": "The Space UUID for your Lightdash instance",
"type": "string"
},
"proxyAuthentication": {
"title": "Proxy Authentication",
"description": "Use if your Lightdash instance is behind a proxy like (Cloud IAP)",
"type": "string",
"format": "password"
},
"supportsMetadataExtraction": {
"title": "Supports Metadata Extraction",
"$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
}
},
"additionalProperties": false,
"required": ["hostPort", "apiKey","projectUUID","spaceUUID"]
}

View File

@ -11,7 +11,7 @@
],
"definitions": {
"dashboardServiceType": {
"description": "Type of Dashboard service - Superset, Looker, Redash, Tableau, Metabase, PowerBi or Mode",
"description": "Type of Dashboard service - Superset, Looker, Redash, Tableau, Metabase, PowerBi, Mode, or Lightdash",
"type": "string",
"javaInterfaces": ["org.openmetadata.schema.EnumInterface"],
"enum": [
@ -25,7 +25,8 @@
"CustomDashboard",
"DomoDashboard",
"QuickSight",
"QlikSense"
"QlikSense",
"Lightdash"
],
"javaEnums": [
{
@ -60,6 +61,9 @@
},
{
"name": "QlikSense"
},
{
"name": "Lightdash"
}
]
},
@ -106,6 +110,9 @@
},
{
"$ref": "./connections/dashboard/qlikSenseConnection.json"
},
{
"$ref": "./connections/dashboard/lightdashConnection.json"
}
]
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

View File

@ -13,6 +13,7 @@
import amazonS3 from 'assets/img/service-icon-amazon-s3.svg';
import gcs from 'assets/img/service-icon-gcs.png';
import lightDash from 'assets/img/service-icon-lightdash.png';
import msAzure from 'assets/img/service-icon-ms-azure.png';
import { EntityType } from 'enums/entity.enum';
import { PipelineType } from 'generated/api/services/ingestionPipelines/createIngestionPipeline';
@ -175,6 +176,7 @@ export const MS_AZURE = msAzure;
export const SPLINE = spline;
export const MONGODB = mongodb;
export const QLIK_SENSE = qlikSense;
export const LIGHT_DASH = lightDash;
export const COUCHBASE = couchbase;
export const PLUS = plus;

View File

@ -19,6 +19,7 @@ import {
} from '../generated/entity/services/dashboardService';
import customDashboardConnection from '../jsons/connectionSchemas/connections/dashboard/customDashboardConnection.json';
import domoDashboardConnection from '../jsons/connectionSchemas/connections/dashboard/domoDashboardConnection.json';
import lightdashConnection from '../jsons/connectionSchemas/connections/dashboard/lightdashConnection.json';
import lookerConnection from '../jsons/connectionSchemas/connections/dashboard/lookerConnection.json';
import metabaseConnection from '../jsons/connectionSchemas/connections/dashboard/metabaseConnection.json';
import modeConnection from '../jsons/connectionSchemas/connections/dashboard/modeConnection.json';
@ -90,11 +91,18 @@ export const getDashboardConfig = (type: DashboardServiceType) => {
break;
}
case DashboardServiceType.QlikSense: {
schema = qliksenseConnection;
break;
}
case DashboardServiceType.Lightdash: {
schema = lightdashConnection;
break;
}
}
return cloneDeep({ schema, uiSchema });

View File

@ -51,6 +51,7 @@ import {
IMPALA,
KAFKA,
KINESIS,
LIGHT_DASH,
LOGO,
LOOKER,
MARIADB,
@ -248,12 +249,16 @@ export const serviceTypeLogo = (type: string) => {
case DashboardServiceType.DomoDashboard:
return DOMO;
case DashboardServiceType.Mode:
return MODE;
case DashboardServiceType.QlikSense:
return QLIK_SENSE;
case DashboardServiceType.Lightdash:
return LIGHT_DASH;
case PipelineServiceType.Airflow:
return AIRFLOW;