mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2026-01-09 05:56:17 +00:00
Mayur Singal
GitHub
parent
5ed85c8923
commit
0c6435ae9b
@ -213,6 +213,7 @@ plugins: Dict[str, Set[str]] = {
|
||||
VERSIONS["packaging"],
|
||||
},
|
||||
"powerbi": {VERSIONS["msal"]},
|
||||
"qliksense": {"websockets~=11.0.3"},
|
||||
"presto": {*COMMONS["hive"]},
|
||||
"pymssql": {"pymssql==2.2.5"},
|
||||
"quicksight": {VERSIONS["boto3"]},
|
||||
|
||||
43
ingestion/src/metadata/examples/workflows/qlik_sense.yaml
Normal file
43
ingestion/src/metadata/examples/workflows/qlik_sense.yaml
Normal file
@ -0,0 +1,43 @@
|
||||
source:
|
||||
type: qliksense
|
||||
serviceName: final_lineage
|
||||
serviceConnection:
|
||||
config:
|
||||
type: QlikSense
|
||||
hostPort: wss://localhost:4747
|
||||
displayUrl: https://localhost
|
||||
certificates:
|
||||
# pass certificate paths
|
||||
clientCertificate: /path/to/client.pem
|
||||
clientKeyCertificate: /path/to/client_key.pem
|
||||
rootCertificate: /path/to/root.pem
|
||||
|
||||
# pass certificate values
|
||||
# clientCertificateData: -----BEGIN CERTIFICATE-----\n....\n.....\n-----END CERTIFICATE-----\n
|
||||
# clientKeyCertificateData: -----BEGIN RSA PRIVATE KEY-----\n....\n....\n-----END RSA PRIVATE KEY-----\n
|
||||
# rootCertificateData: -----BEGIN CERTIFICATE-----\n....\n...-----END CERTIFICATE-----\n
|
||||
# stagingDir: /tmp/stage
|
||||
|
||||
userId: user_id
|
||||
userDirectory: user_dir
|
||||
sourceConfig:
|
||||
config:
|
||||
type: DashboardMetadata
|
||||
dbServiceNames:
|
||||
- mysql
|
||||
- postgres
|
||||
chartFilterPattern: {}
|
||||
# dashboardFilterPattern:
|
||||
# includes:
|
||||
# - .*mysql.*
|
||||
|
||||
sink:
|
||||
type: metadata-rest
|
||||
config: {}
|
||||
workflowConfig:
|
||||
loggerLevel: DEBUG
|
||||
openMetadataServerConfig:
|
||||
hostPort: http://localhost:8585/api
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
|
||||
@ -288,9 +288,28 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
|
||||
f"DataModel is not supported for {self.service_connection.type.name}"
|
||||
)
|
||||
|
||||
def yield_datamodel_dashboard_lineage(
|
||||
self,
|
||||
) -> Iterable[AddLineageRequest]:
|
||||
"""
|
||||
Returns:
|
||||
Lineage request between Data Models and Dashboards
|
||||
"""
|
||||
if hasattr(self.context, "dataModels") and self.context.dataModels:
|
||||
for datamodel in self.context.dataModels:
|
||||
try:
|
||||
yield self._get_add_lineage_request(
|
||||
to_entity=self.context.dashboard, from_entity=datamodel
|
||||
)
|
||||
except Exception as err:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.error(
|
||||
f"Error to yield dashboard lineage details for data model name [{datamodel.name}]: {err}"
|
||||
)
|
||||
|
||||
def yield_dashboard_lineage(
|
||||
self, dashboard_details: Any
|
||||
) -> Optional[Iterable[AddLineageRequest]]:
|
||||
) -> Iterable[AddLineageRequest]:
|
||||
"""
|
||||
Yields lineage if config is enabled.
|
||||
|
||||
@ -301,6 +320,8 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
|
||||
on the dbServiceNames since our lineage will now be
|
||||
model -> dashboard
|
||||
"""
|
||||
yield from self.yield_datamodel_dashboard_lineage() or []
|
||||
|
||||
for db_service_name in self.source_config.dbServiceNames or []:
|
||||
yield from self.yield_dashboard_lineage_details(
|
||||
dashboard_details, db_service_name
|
||||
|
||||
@ -0,0 +1,223 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Websocket Auth & Client for QlikSense
|
||||
"""
|
||||
import json
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from metadata.generated.schema.entity.services.connections.dashboard.qlikSenseConnection import (
|
||||
QlikCertificatePath,
|
||||
QlikCertificateValues,
|
||||
QlikSenseConnection,
|
||||
)
|
||||
from metadata.ingestion.source.dashboard.qliksense.constants import (
|
||||
APP_LOADMODEL_REQ,
|
||||
CREATE_SHEET_SESSION,
|
||||
GET_DOCS_LIST_REQ,
|
||||
GET_LOADMODEL_LAYOUT,
|
||||
GET_SHEET_LAYOUT,
|
||||
OPEN_DOC_REQ,
|
||||
)
|
||||
from metadata.ingestion.source.dashboard.qliksense.models import (
|
||||
QlikDashboard,
|
||||
QlikDashboardResult,
|
||||
QlikDataModelResult,
|
||||
QlikSheet,
|
||||
QlikSheetResult,
|
||||
QlikTable,
|
||||
)
|
||||
from metadata.utils.constants import UTF_8
|
||||
from metadata.utils.helpers import clean_uri, delete_dir_content, init_staging_dir
|
||||
from metadata.utils.logger import ingestion_logger
|
||||
|
||||
logger = ingestion_logger()
|
||||
|
||||
QLIK_USER_HEADER = "X-Qlik-User"
|
||||
|
||||
|
||||
class QlikSenseClient:
|
||||
"""
|
||||
Client Handling API communication with Qlik Engine APIs
|
||||
"""
|
||||
|
||||
def _clean_cert_value(self, cert_data: str) -> str:
|
||||
return cert_data.replace("\\n", "\n")
|
||||
|
||||
def write_data_to_file(self, file_path: Path, cert_data: str) -> None:
|
||||
with open(
|
||||
file_path,
|
||||
"w+",
|
||||
encoding=UTF_8,
|
||||
) as file:
|
||||
data = self._clean_cert_value(cert_data)
|
||||
|
||||
file.write(data)
|
||||
|
||||
def _get_ssl_context(self) -> Optional[dict]:
|
||||
if isinstance(self.config.certificates, QlikCertificatePath):
|
||||
context = {
|
||||
"ca_certs": self.config.certificates.rootCertificate,
|
||||
"certfile": self.config.certificates.clientCertificate,
|
||||
"keyfile": self.config.certificates.clientKeyCertificate,
|
||||
}
|
||||
return context
|
||||
|
||||
init_staging_dir(self.config.certificates.stagingDir)
|
||||
root_path = Path(self.config.certificates.stagingDir, "root.pem")
|
||||
client_path = Path(self.config.certificates.stagingDir, "client.pem")
|
||||
client_key_path = Path(self.config.certificates.stagingDir, "client_key.pem")
|
||||
|
||||
self.write_data_to_file(
|
||||
root_path, self.config.certificates.rootCertificateData.get_secret_value()
|
||||
)
|
||||
|
||||
self.write_data_to_file(
|
||||
client_path,
|
||||
self.config.certificates.clientCertificateData.get_secret_value(),
|
||||
)
|
||||
|
||||
self.write_data_to_file(
|
||||
client_key_path,
|
||||
self.config.certificates.clientKeyCertificateData.get_secret_value(),
|
||||
)
|
||||
|
||||
context = {
|
||||
"ca_certs": root_path,
|
||||
"certfile": client_path,
|
||||
"keyfile": client_key_path,
|
||||
}
|
||||
|
||||
return context
|
||||
|
||||
def connect_websocket(self, app_id: str = None) -> None:
|
||||
"""
|
||||
Method to initialise websocket connection
|
||||
"""
|
||||
import ssl
|
||||
|
||||
from websocket import create_connection
|
||||
|
||||
if self.socket_connection:
|
||||
self.socket_connection.close()
|
||||
|
||||
ssl_conext = self._get_ssl_context()
|
||||
ssl.match_hostname = lambda cert, hostname: True
|
||||
self.socket_connection = create_connection(
|
||||
f"{clean_uri(self.config.hostPort)}/app/{app_id or ''}",
|
||||
sslopt=ssl_conext,
|
||||
header={
|
||||
f"{QLIK_USER_HEADER}: "
|
||||
f"UserDirectory={self.config.userDirectory}; UserId={self.config.userId}"
|
||||
},
|
||||
)
|
||||
if app_id:
|
||||
# get doc list needs to be executed before extracting data from app
|
||||
self.get_dashboards_list(create_new_socket=False)
|
||||
|
||||
def close_websocket(self) -> None:
|
||||
if self.socket_connection:
|
||||
self.socket_connection.close()
|
||||
|
||||
if isinstance(self.config.certificates, QlikCertificateValues):
|
||||
delete_dir_content(self.config.certificates.stagingDir)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: QlikSenseConnection,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.socket_connection = None
|
||||
|
||||
def _websocket_send_request(
|
||||
self, request: dict, response: bool = False
|
||||
) -> Optional[Dict]:
|
||||
"""
|
||||
Method to send request to websocket
|
||||
|
||||
request: data required to be sent to websocket
|
||||
response: is json response required?
|
||||
"""
|
||||
self.socket_connection.send(json.dumps(request))
|
||||
resp = self.socket_connection.recv()
|
||||
if response:
|
||||
return json.loads(resp)
|
||||
return None
|
||||
|
||||
def get_dashboards_list(
|
||||
self, create_new_socket: bool = True
|
||||
) -> List[QlikDashboard]:
|
||||
"""
|
||||
Get List of all dashboards
|
||||
"""
|
||||
try:
|
||||
if create_new_socket:
|
||||
self.connect_websocket()
|
||||
self._websocket_send_request(GET_DOCS_LIST_REQ)
|
||||
resp = self.socket_connection.recv()
|
||||
dashboard_result = QlikDashboardResult(**json.loads(resp))
|
||||
return dashboard_result.result.qDocList
|
||||
except Exception:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning("Failed to fetch the dashboard list")
|
||||
return []
|
||||
|
||||
def get_dashboard_charts(self, dashboard_id: str) -> List[QlikSheet]:
|
||||
"""
|
||||
Get dahsboard chart list
|
||||
"""
|
||||
try:
|
||||
OPEN_DOC_REQ.update({"params": [dashboard_id]})
|
||||
self._websocket_send_request(OPEN_DOC_REQ)
|
||||
self._websocket_send_request(CREATE_SHEET_SESSION)
|
||||
sheets = self._websocket_send_request(GET_SHEET_LAYOUT, response=True)
|
||||
data = QlikSheetResult(**sheets)
|
||||
return data.result.qLayout.qAppObjectList.qItems
|
||||
except Exception:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning("Failed to fetch the dashboard charts")
|
||||
return []
|
||||
|
||||
def get_dashboard_models(self) -> List[QlikTable]:
|
||||
"""
|
||||
Get dahsboard chart list
|
||||
"""
|
||||
try:
|
||||
self._websocket_send_request(APP_LOADMODEL_REQ)
|
||||
models = self._websocket_send_request(GET_LOADMODEL_LAYOUT, response=True)
|
||||
data_models = QlikDataModelResult(**models)
|
||||
layout = data_models.result.qLayout
|
||||
if isinstance(layout, list):
|
||||
tables = []
|
||||
for layout in data_models.result.qLayout:
|
||||
tables.extend(layout.value.tables)
|
||||
return tables
|
||||
return layout.tables
|
||||
except Exception:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning("Failed to fetch the dashboard datamodels")
|
||||
return []
|
||||
|
||||
def get_dashboard_for_test_connection(self):
|
||||
try:
|
||||
self.connect_websocket()
|
||||
self._websocket_send_request(GET_DOCS_LIST_REQ)
|
||||
resp = self.socket_connection.recv()
|
||||
self.close_websocket()
|
||||
return QlikDashboardResult(**json.loads(resp))
|
||||
except ValidationError:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning("Failed to fetch the dashboard datamodels")
|
||||
return None
|
||||
@ -0,0 +1,53 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Source connection handler
|
||||
"""
|
||||
from typing import Optional
|
||||
|
||||
from metadata.generated.schema.entity.automations.workflow import (
|
||||
Workflow as AutomationWorkflow,
|
||||
)
|
||||
from metadata.generated.schema.entity.services.connections.dashboard.qlikSenseConnection import (
|
||||
QlikSenseConnection,
|
||||
)
|
||||
from metadata.ingestion.connections.test_connections import test_connection_steps
|
||||
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
||||
from metadata.ingestion.source.dashboard.qliksense.client import QlikSenseClient
|
||||
|
||||
|
||||
def get_connection(connection: QlikSenseConnection) -> QlikSenseClient:
|
||||
"""
|
||||
Create connection
|
||||
"""
|
||||
return QlikSenseClient(connection)
|
||||
|
||||
|
||||
def test_connection(
|
||||
metadata: OpenMetadata,
|
||||
client: QlikSenseClient,
|
||||
service_connection: QlikSenseConnection,
|
||||
automation_workflow: Optional[AutomationWorkflow] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Test connection. This can be executed either as part
|
||||
of a metadata workflow or during an Automation Workflow
|
||||
"""
|
||||
|
||||
test_fn = {"GetDashboards": client.get_dashboard_for_test_connection}
|
||||
|
||||
test_connection_steps(
|
||||
metadata=metadata,
|
||||
test_fn=test_fn,
|
||||
service_type=service_connection.type.value,
|
||||
automation_workflow=automation_workflow,
|
||||
)
|
||||
@ -0,0 +1,80 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
QlikSense Constants
|
||||
"""
|
||||
|
||||
|
||||
GET_DOCS_LIST_REQ = {
|
||||
"handle": -1,
|
||||
"method": "GetDocList",
|
||||
"params": [],
|
||||
"outKey": -1,
|
||||
"id": 1,
|
||||
}
|
||||
|
||||
OPEN_DOC_REQ = {
|
||||
"method": "OpenDoc",
|
||||
"handle": -1,
|
||||
"outKey": -1,
|
||||
"id": 2,
|
||||
}
|
||||
|
||||
CREATE_SHEET_SESSION = {
|
||||
"method": "CreateSessionObject",
|
||||
"handle": 1,
|
||||
"params": [
|
||||
{
|
||||
"qInfo": {"qType": "SheetList"},
|
||||
"qAppObjectListDef": {
|
||||
"qType": "sheet",
|
||||
"qData": {
|
||||
"title": "/qMetaDef/title",
|
||||
"description": "/qMetaDef/description",
|
||||
"thumbnail": "/thumbnail",
|
||||
"cells": "/cells",
|
||||
"rank": "/rank",
|
||||
"columns": "/columns",
|
||||
"rows": "/rows",
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"outKey": -1,
|
||||
"id": 3,
|
||||
}
|
||||
|
||||
GET_SHEET_LAYOUT = {
|
||||
"method": "GetLayout",
|
||||
"handle": 2,
|
||||
"params": [],
|
||||
"outKey": -1,
|
||||
"id": 4,
|
||||
}
|
||||
|
||||
APP_LOADMODEL_REQ = {
|
||||
"delta": True,
|
||||
"handle": 1,
|
||||
"method": "GetObject",
|
||||
"params": ["LoadModel"],
|
||||
"id": 5,
|
||||
"jsonrpc": "2.0",
|
||||
}
|
||||
|
||||
|
||||
GET_LOADMODEL_LAYOUT = {
|
||||
"delta": True,
|
||||
"handle": 3,
|
||||
"method": "GetLayout",
|
||||
"params": [],
|
||||
"id": 6,
|
||||
"jsonrpc": "2.0",
|
||||
}
|
||||
@ -0,0 +1,327 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Qlik Sense Source Module"""
|
||||
|
||||
import traceback
|
||||
from typing import Iterable, List, Optional
|
||||
|
||||
from metadata.generated.schema.api.data.createChart import CreateChartRequest
|
||||
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
|
||||
from metadata.generated.schema.api.data.createDashboardDataModel import (
|
||||
CreateDashboardDataModelRequest,
|
||||
)
|
||||
from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
|
||||
from metadata.generated.schema.entity.data.chart import Chart, ChartType
|
||||
from metadata.generated.schema.entity.data.dashboardDataModel import (
|
||||
DashboardDataModel,
|
||||
DataModelType,
|
||||
)
|
||||
from metadata.generated.schema.entity.data.table import Column, DataType, Table
|
||||
from metadata.generated.schema.entity.services.connections.dashboard.qlikSenseConnection import (
|
||||
QlikSenseConnection,
|
||||
)
|
||||
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
|
||||
OpenMetadataConnection,
|
||||
)
|
||||
from metadata.generated.schema.entity.services.dashboardService import (
|
||||
DashboardServiceType,
|
||||
)
|
||||
from metadata.generated.schema.entity.services.databaseService import DatabaseService
|
||||
from metadata.generated.schema.metadataIngestion.workflow import (
|
||||
Source as WorkflowSource,
|
||||
)
|
||||
from metadata.ingestion.api.source import InvalidSourceException
|
||||
from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
|
||||
from metadata.ingestion.source.dashboard.qliksense.client import QlikSenseClient
|
||||
from metadata.ingestion.source.dashboard.qliksense.models import (
|
||||
QlikDashboard,
|
||||
QlikTable,
|
||||
)
|
||||
from metadata.utils import fqn
|
||||
from metadata.utils.filters import filter_by_chart, filter_by_datamodel
|
||||
from metadata.utils.helpers import clean_uri
|
||||
from metadata.utils.logger import ingestion_logger
|
||||
|
||||
logger = ingestion_logger()
|
||||
|
||||
|
||||
class QliksenseSource(DashboardServiceSource):
|
||||
"""
|
||||
Qlik Sense Source Class
|
||||
"""
|
||||
|
||||
config: WorkflowSource
|
||||
client: QlikSenseClient
|
||||
metadata_config: OpenMetadataConnection
|
||||
|
||||
@classmethod
|
||||
def create(cls, config_dict, metadata_config: OpenMetadataConnection):
|
||||
config = WorkflowSource.parse_obj(config_dict)
|
||||
connection: QlikSenseConnection = config.serviceConnection.__root__.config
|
||||
if not isinstance(connection, QlikSenseConnection):
|
||||
raise InvalidSourceException(
|
||||
f"Expected QlikSenseConnection, but got {connection}"
|
||||
)
|
||||
return cls(config, metadata_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WorkflowSource,
|
||||
metadata_config: OpenMetadataConnection,
|
||||
):
|
||||
super().__init__(config, metadata_config)
|
||||
self.collections: List[QlikDashboard] = []
|
||||
|
||||
def get_dashboards_list(self) -> Iterable[QlikDashboard]:
|
||||
"""
|
||||
Get List of all dashboards
|
||||
"""
|
||||
for dashboard in self.client.get_dashboards_list():
|
||||
# create app specific websocket
|
||||
self.client.connect_websocket(dashboard.qDocId)
|
||||
# clean data models for next iteration
|
||||
self.data_models = []
|
||||
yield dashboard
|
||||
|
||||
def get_dashboard_name(self, dashboard: QlikDashboard) -> str:
|
||||
"""
|
||||
Get Dashboard Name
|
||||
"""
|
||||
return dashboard.qDocName
|
||||
|
||||
def get_dashboard_details(self, dashboard: QlikDashboard) -> dict:
|
||||
"""
|
||||
Get Dashboard Details
|
||||
"""
|
||||
return dashboard
|
||||
|
||||
def yield_dashboard(
|
||||
self, dashboard_details: QlikDashboard
|
||||
) -> Iterable[CreateDashboardRequest]:
|
||||
"""
|
||||
Method to Get Dashboard Entity
|
||||
"""
|
||||
try:
|
||||
if self.service_connection.displayUrl:
|
||||
dashboard_url = (
|
||||
f"{clean_uri(self.service_connection.displayUrl)}/sense/app/"
|
||||
f"{dashboard_details.qDocId}/overview"
|
||||
)
|
||||
else:
|
||||
dashboard_url = None
|
||||
|
||||
dashboard_request = CreateDashboardRequest(
|
||||
name=dashboard_details.qDocId,
|
||||
sourceUrl=dashboard_url,
|
||||
displayName=dashboard_details.qDocName,
|
||||
description=dashboard_details.qMeta.description,
|
||||
charts=[
|
||||
fqn.build(
|
||||
self.metadata,
|
||||
entity_type=Chart,
|
||||
service_name=self.context.dashboard_service.fullyQualifiedName.__root__,
|
||||
chart_name=chart.name.__root__,
|
||||
)
|
||||
for chart in self.context.charts
|
||||
],
|
||||
service=self.context.dashboard_service.fullyQualifiedName.__root__,
|
||||
)
|
||||
yield dashboard_request
|
||||
self.register_record(dashboard_request=dashboard_request)
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning(
|
||||
f"Error creating dashboard [{dashboard_details.qDocName}]: {exc}"
|
||||
)
|
||||
|
||||
def yield_dashboard_chart(
|
||||
self, dashboard_details: QlikDashboard
|
||||
) -> Iterable[CreateChartRequest]:
|
||||
"""Get chart method
|
||||
|
||||
Args:
|
||||
dashboard_details:
|
||||
Returns:
|
||||
Iterable[CreateChartRequest]
|
||||
"""
|
||||
charts = self.client.get_dashboard_charts(dashboard_id=dashboard_details.qDocId)
|
||||
for chart in charts:
|
||||
try:
|
||||
if not chart.qInfo.qId:
|
||||
continue
|
||||
if self.service_connection.displayUrl:
|
||||
chart_url = (
|
||||
f"{clean_uri(self.service_connection.displayUrl)}/sense/app/{dashboard_details.qDocId}"
|
||||
f"/sheet/{chart.qInfo.qId}"
|
||||
)
|
||||
else:
|
||||
chart_url = None
|
||||
if chart.qMeta.title and filter_by_chart(
|
||||
self.source_config.chartFilterPattern, chart.qMeta.title
|
||||
):
|
||||
self.status.filter(chart.qMeta.title, "Chart Pattern not allowed")
|
||||
continue
|
||||
yield CreateChartRequest(
|
||||
name=chart.qInfo.qId,
|
||||
displayName=chart.qMeta.title,
|
||||
description=chart.qMeta.description,
|
||||
chartType=ChartType.Other,
|
||||
sourceUrl=chart_url,
|
||||
service=self.context.dashboard_service.fullyQualifiedName.__root__,
|
||||
)
|
||||
self.status.scanned(chart.qMeta.title)
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning(f"Error creating chart [{chart}]: {exc}")
|
||||
|
||||
def get_column_info(self, data_source: QlikTable) -> Optional[List[Column]]:
|
||||
"""
|
||||
Args:
|
||||
data_source: DataSource
|
||||
Returns:
|
||||
Columns details for Data Model
|
||||
"""
|
||||
datasource_columns = []
|
||||
for field in data_source.fields or []:
|
||||
try:
|
||||
parsed_fields = {
|
||||
"dataTypeDisplay": "Qlik Field",
|
||||
"dataType": DataType.UNKNOWN,
|
||||
"name": field.id,
|
||||
"displayName": field.name if field.name else field.id,
|
||||
}
|
||||
datasource_columns.append(Column(**parsed_fields))
|
||||
except Exception as exc:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning(f"Error to yield datamodel column: {exc}")
|
||||
return datasource_columns
|
||||
|
||||
def yield_datamodel(self, dashboard_details: QlikDashboard):
|
||||
if self.source_config.includeDataModels:
|
||||
self.data_models = self.client.get_dashboard_models()
|
||||
for data_model in self.data_models or []:
|
||||
try:
|
||||
data_model_name = (
|
||||
data_model.tableName if data_model.tableName else data_model.id
|
||||
)
|
||||
if filter_by_datamodel(
|
||||
self.source_config.dataModelFilterPattern, data_model_name
|
||||
):
|
||||
self.status.filter(data_model_name, "Data model filtered out.")
|
||||
continue
|
||||
|
||||
data_model_request = CreateDashboardDataModelRequest(
|
||||
name=data_model.id,
|
||||
displayName=data_model_name,
|
||||
service=self.context.dashboard_service.fullyQualifiedName.__root__,
|
||||
dataModelType=DataModelType.QlikSenseDataModel.value,
|
||||
serviceType=DashboardServiceType.QlikSense.value,
|
||||
columns=self.get_column_info(data_model),
|
||||
)
|
||||
yield data_model_request
|
||||
self.status.scanned(
|
||||
f"Data Model Scanned: {data_model_request.displayName}"
|
||||
)
|
||||
except Exception as exc:
|
||||
error_msg = f"Error yielding Data Model [{data_model_name}]: {exc}"
|
||||
self.status.failed(
|
||||
name=data_model_name,
|
||||
error=error_msg,
|
||||
stack_trace=traceback.format_exc(),
|
||||
)
|
||||
logger.error(error_msg)
|
||||
logger.debug(traceback.format_exc())
|
||||
|
||||
def _get_datamodel(self, datamodel: QlikTable):
|
||||
datamodel_fqn = fqn.build(
|
||||
self.metadata,
|
||||
entity_type=DashboardDataModel,
|
||||
service_name=self.context.dashboard_service.fullyQualifiedName.__root__,
|
||||
data_model_name=datamodel.id,
|
||||
)
|
||||
if datamodel_fqn:
|
||||
return self.metadata.get_by_name(
|
||||
entity=DashboardDataModel,
|
||||
fqn=datamodel_fqn,
|
||||
)
|
||||
return None
|
||||
|
||||
def _get_database_table(
|
||||
self, db_service_entity: DatabaseService, datamodel: QlikTable
|
||||
) -> Optional[Table]:
|
||||
"""
|
||||
Get the table entity for lineage
|
||||
"""
|
||||
# table.name in tableau can come as db.schema.table_name. Hence the logic to split it
|
||||
if datamodel.tableName and db_service_entity:
|
||||
try:
|
||||
if len(datamodel.connectorProperties.tableQualifiers) > 1:
|
||||
(
|
||||
database_name,
|
||||
schema_name,
|
||||
) = datamodel.connectorProperties.tableQualifiers[-2:]
|
||||
elif len(datamodel.connectorProperties.tableQualifiers) == 1:
|
||||
schema_name = datamodel.connectorProperties.tableQualifiers[-1]
|
||||
database_name = None
|
||||
else:
|
||||
schema_name, database_name = None, None
|
||||
|
||||
table_fqn = fqn.build(
|
||||
self.metadata,
|
||||
entity_type=Table,
|
||||
service_name=db_service_entity.name.__root__,
|
||||
schema_name=schema_name,
|
||||
table_name=datamodel.tableName,
|
||||
database_name=database_name,
|
||||
)
|
||||
if table_fqn:
|
||||
return self.metadata.get_by_name(
|
||||
entity=Table,
|
||||
fqn=table_fqn,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning(f"Error occured while finding table fqn: {exc}")
|
||||
return None
|
||||
|
||||
def yield_dashboard_lineage_details(
|
||||
self,
|
||||
dashboard_details: QlikDashboard,
|
||||
db_service_name: Optional[str],
|
||||
) -> Iterable[AddLineageRequest]:
|
||||
"""Get lineage method
|
||||
|
||||
Args:
|
||||
dashboard_details
|
||||
"""
|
||||
db_service_entity = self.metadata.get_by_name(
|
||||
entity=DatabaseService, fqn=db_service_name
|
||||
)
|
||||
for datamodel in self.data_models or []:
|
||||
try:
|
||||
data_model_entity = self._get_datamodel(datamodel=datamodel)
|
||||
if data_model_entity:
|
||||
om_table = self._get_database_table(
|
||||
db_service_entity, datamodel=datamodel
|
||||
)
|
||||
if om_table:
|
||||
yield self._get_add_lineage_request(
|
||||
to_entity=data_model_entity, from_entity=om_table
|
||||
)
|
||||
except Exception as err:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.error(
|
||||
f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {err}"
|
||||
)
|
||||
|
||||
def close(self):
|
||||
self.client.close_websocket()
|
||||
return super().close()
|
||||
@ -0,0 +1,119 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
QlikSense Models
|
||||
"""
|
||||
from typing import List, Optional, Union
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
# dashboard models
|
||||
|
||||
|
||||
class QlikDashboardMeta(BaseModel):
|
||||
description: Optional[str]
|
||||
|
||||
|
||||
class QlikDashboard(BaseModel):
|
||||
qDocName: str
|
||||
qDocId: str
|
||||
qTitle: str
|
||||
qMeta: Optional[QlikDashboardMeta] = QlikDashboardMeta()
|
||||
|
||||
|
||||
class QlikDashboardList(BaseModel):
|
||||
qDocList: Optional[List[QlikDashboard]] = []
|
||||
|
||||
|
||||
class QlikDashboardResult(BaseModel):
|
||||
result: Optional[QlikDashboardList] = QlikDashboardList()
|
||||
|
||||
|
||||
# sheet models
|
||||
class QlikSheetInfo(BaseModel):
|
||||
qId: str
|
||||
|
||||
|
||||
class QlikSheetMeta(BaseModel):
|
||||
title: Optional[str]
|
||||
description: Optional[str]
|
||||
|
||||
|
||||
class QlikSheet(BaseModel):
|
||||
qInfo: QlikSheetInfo
|
||||
qMeta: Optional[QlikSheetMeta] = QlikSheetMeta()
|
||||
|
||||
|
||||
class QlikSheetItems(BaseModel):
|
||||
qItems: Optional[List[QlikSheet]] = []
|
||||
|
||||
|
||||
class QlikSheetAppObject(BaseModel):
|
||||
qAppObjectList: Optional[QlikSheetItems] = QlikSheetItems()
|
||||
|
||||
|
||||
class QlikSheetLayout(BaseModel):
|
||||
qLayout: Optional[QlikSheetAppObject] = QlikSheetAppObject()
|
||||
|
||||
|
||||
class QlikSheetResult(BaseModel):
|
||||
result: Optional[QlikSheetLayout] = QlikSheetLayout()
|
||||
|
||||
|
||||
# datamodel models
|
||||
class QlikFields(BaseModel):
|
||||
name: Optional[str]
|
||||
id: Optional[str]
|
||||
|
||||
|
||||
class QlikTableConnectionProp(BaseModel):
|
||||
tableQualifiers: Optional[List[str]] = []
|
||||
|
||||
|
||||
class QlikTable(BaseModel):
|
||||
tableName: Optional[str]
|
||||
id: Optional[str]
|
||||
connectorProperties: Optional[QlikTableConnectionProp] = QlikTableConnectionProp()
|
||||
fields: Optional[List[QlikFields]] = []
|
||||
|
||||
|
||||
class QlikTablesList(BaseModel):
|
||||
tables: Optional[List[QlikTable]] = []
|
||||
|
||||
|
||||
class QlikDataModelValue(BaseModel):
|
||||
value: Optional[QlikTablesList] = QlikTablesList()
|
||||
|
||||
|
||||
class QlikDataModelLayout(BaseModel):
|
||||
qLayout: Optional[
|
||||
Union[QlikTablesList, List[QlikDataModelValue]]
|
||||
] = QlikTablesList()
|
||||
|
||||
|
||||
class QlikDataModelResult(BaseModel):
|
||||
result: Optional[QlikDataModelLayout] = QlikDataModelLayout()
|
||||
|
||||
|
||||
class QlikLayoutHandle(BaseModel):
|
||||
qHandle: Optional[int] = 2
|
||||
|
||||
|
||||
class QlikLayoutValue(BaseModel):
|
||||
value: Optional[QlikLayoutHandle] = QlikLayoutHandle()
|
||||
|
||||
|
||||
class QlikQReturn(BaseModel):
|
||||
qReturn: Optional[Union[QlikLayoutHandle, List[QlikLayoutValue]]] = []
|
||||
|
||||
|
||||
class QlikLayoutResult(BaseModel):
|
||||
result: Optional[QlikQReturn] = QlikQReturn()
|
||||
@ -224,31 +224,3 @@ class SupersetSourceMixin(DashboardServiceSource):
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning(f"Error to yield datamodel column: {exc}")
|
||||
return datasource_columns
|
||||
|
||||
def yield_dashboard_lineage(
|
||||
self, dashboard_details: Union[FetchDashboard, DashboradResult]
|
||||
) -> Optional[Iterable[AddLineageRequest]]:
|
||||
yield from self.yield_datamodel_dashboard_lineage() or []
|
||||
|
||||
for db_service_name in self.source_config.dbServiceNames or []:
|
||||
yield from self.yield_dashboard_lineage_details(
|
||||
dashboard_details, db_service_name
|
||||
) or []
|
||||
|
||||
def yield_datamodel_dashboard_lineage(
|
||||
self,
|
||||
) -> Optional[Iterable[AddLineageRequest]]:
|
||||
"""
|
||||
Returns:
|
||||
Lineage request between Data Models and Dashboards
|
||||
"""
|
||||
for datamodel in self.context.dataModels or []:
|
||||
try:
|
||||
yield self._get_add_lineage_request(
|
||||
to_entity=self.context.dashboard, from_entity=datamodel
|
||||
)
|
||||
except Exception as err:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.error(
|
||||
f"Error to yield dashboard lineage details for data model name [{datamodel.name}]: {err}"
|
||||
)
|
||||
|
||||
@ -266,34 +266,6 @@ class TableauSource(DashboardServiceSource):
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning(f"Error to yield dashboard for {dashboard_details}: {exc}")
|
||||
|
||||
def yield_dashboard_lineage(
|
||||
self, dashboard_details: TableauDashboard
|
||||
) -> Optional[Iterable[AddLineageRequest]]:
|
||||
yield from self.yield_datamodel_dashboard_lineage() or []
|
||||
|
||||
for db_service_name in self.source_config.dbServiceNames or []:
|
||||
yield from self.yield_dashboard_lineage_details(
|
||||
dashboard_details, db_service_name
|
||||
) or []
|
||||
|
||||
def yield_datamodel_dashboard_lineage(
|
||||
self,
|
||||
) -> Optional[Iterable[AddLineageRequest]]:
|
||||
"""
|
||||
Returns:
|
||||
Lineage request between Data Models and Dashboards
|
||||
"""
|
||||
for datamodel in self.context.dataModels or []:
|
||||
try:
|
||||
yield self._get_add_lineage_request(
|
||||
to_entity=self.context.dashboard, from_entity=datamodel
|
||||
)
|
||||
except Exception as err:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.error(
|
||||
f"Error to yield dashboard lineage details for data model name [{datamodel.name}]: {err}"
|
||||
)
|
||||
|
||||
def yield_dashboard_lineage_details(
|
||||
self, dashboard_details: TableauDashboard, db_service_name: str
|
||||
) -> Optional[Iterable[AddLineageRequest]]:
|
||||
|
||||
@ -30,6 +30,7 @@ from metadata.generated.schema.type.tableUsageCount import TableUsageCount
|
||||
from metadata.ingestion.api.stage import Stage
|
||||
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
||||
from metadata.utils.constants import UTF_8
|
||||
from metadata.utils.helpers import init_staging_dir
|
||||
from metadata.utils.logger import ingestion_logger
|
||||
|
||||
logger = ingestion_logger()
|
||||
@ -60,9 +61,7 @@ class TableUsageStage(Stage[QueryParserData]):
|
||||
self.metadata = OpenMetadata(self.metadata_config)
|
||||
self.table_usage = {}
|
||||
self.table_queries = {}
|
||||
|
||||
self.init_location()
|
||||
|
||||
init_staging_dir(self.config.filename)
|
||||
self.wrote_something = False
|
||||
|
||||
@classmethod
|
||||
|
||||
@ -17,10 +17,12 @@ from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
from functools import wraps
|
||||
from math import floor, log
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
|
||||
@ -455,3 +457,20 @@ def get_database_name_for_lineage(
|
||||
db_service_entity.connection.config.__dict__.get("databaseName")
|
||||
or DEFAULT_DATABASE
|
||||
)
|
||||
|
||||
|
||||
def delete_dir_content(directory: str) -> None:
|
||||
location = Path(directory)
|
||||
if location.is_dir():
|
||||
logger.info("Location exists, cleaning it up")
|
||||
shutil.rmtree(directory)
|
||||
|
||||
|
||||
def init_staging_dir(directory: str) -> None:
|
||||
"""
|
||||
Prepare the the staging directory
|
||||
"""
|
||||
delete_dir_content(directory=directory)
|
||||
location = Path(directory)
|
||||
logger.info(f"Creating the directory to store staging data in {location}")
|
||||
location.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
186
ingestion/tests/unit/topology/dashboard/test_qliksense.py
Normal file
186
ingestion/tests/unit/topology/dashboard/test_qliksense.py
Normal file
@ -0,0 +1,186 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Test QlikSense using the topology
|
||||
"""
|
||||
|
||||
from unittest import TestCase
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from metadata.generated.schema.api.data.createChart import CreateChartRequest
|
||||
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
|
||||
from metadata.generated.schema.entity.services.dashboardService import (
|
||||
DashboardConnection,
|
||||
DashboardService,
|
||||
DashboardServiceType,
|
||||
)
|
||||
from metadata.generated.schema.metadataIngestion.workflow import (
|
||||
OpenMetadataWorkflowConfig,
|
||||
)
|
||||
from metadata.generated.schema.type.basic import FullyQualifiedEntityName
|
||||
from metadata.ingestion.source.dashboard.qliksense.client import QlikSenseClient
|
||||
from metadata.ingestion.source.dashboard.qliksense.metadata import QliksenseSource
|
||||
from metadata.ingestion.source.dashboard.qliksense.models import (
|
||||
QlikDashboard,
|
||||
QlikSheet,
|
||||
QlikSheetInfo,
|
||||
QlikSheetMeta,
|
||||
)
|
||||
|
||||
MOCK_DASHBOARD_SERVICE = DashboardService(
|
||||
id="c3eb265f-5445-4ad3-ba5e-797d3a3071bb",
|
||||
name="qliksense_source_test",
|
||||
fullyQualifiedName=FullyQualifiedEntityName(__root__="qliksense_source_test"),
|
||||
connection=DashboardConnection(),
|
||||
serviceType=DashboardServiceType.QlikSense,
|
||||
)
|
||||
|
||||
|
||||
mock_qliksense_config = {
|
||||
"source": {
|
||||
"type": "qliksense",
|
||||
"serviceName": "local_qliksensem",
|
||||
"serviceConnection": {
|
||||
"config": {
|
||||
"type": "QlikSense",
|
||||
"certificates": {
|
||||
"rootCertificate": "/test/path/root.pem",
|
||||
"clientKeyCertificate": "/test/path/client_key.pem",
|
||||
"clientCertificate": "/test/path/client.pem",
|
||||
},
|
||||
"userDirectory": "demo",
|
||||
"userId": "demo",
|
||||
"hostPort": "wss://test:4747",
|
||||
"displayUrl": "https://test",
|
||||
}
|
||||
},
|
||||
"sourceConfig": {
|
||||
"config": {"dashboardFilterPattern": {}, "chartFilterPattern": {}}
|
||||
},
|
||||
},
|
||||
"sink": {"type": "metadata-rest", "config": {}},
|
||||
"workflowConfig": {
|
||||
"openMetadataServerConfig": {
|
||||
"hostPort": "http://localhost:8585/api",
|
||||
"authProvider": "openmetadata",
|
||||
"securityConfig": {
|
||||
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
MOCK_DASHBOARD_NAME = "New Dashboard"
|
||||
|
||||
MOCK_DASHBOARD_DETAILS = QlikDashboard(
|
||||
qDocName=MOCK_DASHBOARD_NAME,
|
||||
qDocId="1",
|
||||
qTitle=MOCK_DASHBOARD_NAME,
|
||||
)
|
||||
|
||||
MOCK_CHARTS = [
|
||||
QlikSheet(
|
||||
qInfo=QlikSheetInfo(qId="11"), qMeta=QlikSheetMeta(title="Top Salespeople")
|
||||
),
|
||||
QlikSheet(
|
||||
qInfo=QlikSheetInfo(qId="12"),
|
||||
qMeta=QlikSheetMeta(title="Milan Datasets", description="dummy"),
|
||||
),
|
||||
]
|
||||
|
||||
EXPECTED_DASHBOARD = CreateDashboardRequest(
|
||||
name="1",
|
||||
displayName="New Dashboard",
|
||||
sourceUrl="https://test/sense/app/1/overview",
|
||||
charts=[],
|
||||
tags=None,
|
||||
owner=None,
|
||||
service="qliksense_source_test",
|
||||
extension=None,
|
||||
)
|
||||
|
||||
EXPECTED_DASHBOARDS = [
|
||||
CreateChartRequest(
|
||||
name="11",
|
||||
displayName="Top Salespeople",
|
||||
chartType="Other",
|
||||
sourceUrl="https://test/sense/app/1/sheet/11",
|
||||
tags=None,
|
||||
owner=None,
|
||||
service="qliksense_source_test",
|
||||
),
|
||||
CreateChartRequest(
|
||||
name="12",
|
||||
displayName="Milan Datasets",
|
||||
chartType="Other",
|
||||
sourceUrl="https://test/sense/app/1/sheet/12",
|
||||
tags=None,
|
||||
owner=None,
|
||||
service="qliksense_source_test",
|
||||
description="dummy",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class QlikSenseUnitTest(TestCase):
|
||||
"""
|
||||
Implements the necessary methods to extract
|
||||
QlikSense Unit Test
|
||||
"""
|
||||
|
||||
def __init__(self, methodName) -> None:
|
||||
with patch.object(
|
||||
QlikSenseClient, "get_dashboard_for_test_connection", return_value=None
|
||||
):
|
||||
super().__init__(methodName)
|
||||
# test_connection.return_value = False
|
||||
self.config = OpenMetadataWorkflowConfig.parse_obj(mock_qliksense_config)
|
||||
self.qliksense = QliksenseSource.create(
|
||||
mock_qliksense_config["source"],
|
||||
self.config.workflowConfig.openMetadataServerConfig,
|
||||
)
|
||||
self.qliksense.context.__dict__[
|
||||
"dashboard_service"
|
||||
] = MOCK_DASHBOARD_SERVICE
|
||||
|
||||
@pytest.mark.order(1)
|
||||
def test_dashboard(self):
|
||||
dashboard_list = []
|
||||
results = self.qliksense.yield_dashboard(MOCK_DASHBOARD_DETAILS)
|
||||
for result in results:
|
||||
if isinstance(result, CreateDashboardRequest):
|
||||
dashboard_list.append(result)
|
||||
self.assertEqual(EXPECTED_DASHBOARD, dashboard_list[0])
|
||||
|
||||
@pytest.mark.order(2)
|
||||
def test_dashboard_name(self):
|
||||
assert (
|
||||
self.qliksense.get_dashboard_name(MOCK_DASHBOARD_DETAILS)
|
||||
== MOCK_DASHBOARD_NAME
|
||||
)
|
||||
|
||||
@pytest.mark.order(3)
|
||||
def test_chart(self):
|
||||
dashboard_details = MOCK_DASHBOARD_DETAILS
|
||||
with patch.object(
|
||||
QlikSenseClient, "get_dashboard_charts", return_value=MOCK_CHARTS
|
||||
):
|
||||
results = list(self.qliksense.yield_dashboard_chart(dashboard_details))
|
||||
chart_list = []
|
||||
for result in results:
|
||||
if isinstance(result, CreateChartRequest):
|
||||
chart_list.append(result)
|
||||
for _, (expected, original) in enumerate(
|
||||
zip(EXPECTED_DASHBOARDS, chart_list)
|
||||
):
|
||||
self.assertEqual(expected, original)
|
||||
@ -16,6 +16,7 @@ This is the supported list of connectors for Dashboard Services:
|
||||
- [Redash](/connectors/dashboard/redash)
|
||||
- [Superset](/connectors/dashboard/superset)
|
||||
- [Tableau](/connectors/dashboard/tableau)
|
||||
- [Qlik Sense](/connectors/dashboard/qliksense)
|
||||
|
||||
If you have a request for a new connector, don't hesitate to reach out in [Slack](https://slack.open-metadata.org/) or
|
||||
open a [feature request](https://github.com/open-metadata/OpenMetadata/issues/new/choose) in our GitHub repo.
|
||||
|
||||
@ -0,0 +1,52 @@
|
||||
---
|
||||
title: Qlik Sense
|
||||
slug: /connectors/dashboard/qliksense/certificates
|
||||
---
|
||||
|
||||
# How to generate authentication certificates
|
||||
|
||||
OpenMetadata Uses [Qlik Engine APIs](https://help.qlik.com/en-US/sense-developer/May2023/Subsystems/EngineAPI/Content/Sense_EngineAPI/introducing-engine-API.htm) to communicate with Qlik Sense and fetch relevant metadata, and connecting to these APIs require authentication certificates as described in [these docs](https://help.qlik.com/en-US/sense-developer/May2023/Subsystems/EngineAPI/Content/Sense_EngineAPI/GettingStarted/connecting-to-engine-api.htm).
|
||||
|
||||
|
||||
In this document we will explain how you can generate these certificates so that OpenMetadata can communicate with Qlik Sense.
|
||||
|
||||
|
||||
# Step 1: Open Qlik Management Console (QMC)
|
||||
|
||||
Open your Qlik Management Console (QMC) and navigate to certificates section.
|
||||
|
||||
{% image
|
||||
src="/images/v1.1.1/connectors/qliksense/qlik-certificate-nav.png"
|
||||
alt="Navigate to certificates in QMC"
|
||||
caption="Navigate to certificates in QMC"
|
||||
/%}
|
||||
|
||||
# Step 2: Provide Details and Export Certificates
|
||||
|
||||
1. In the Machine name box, type the full computer name of the computer that you are creating the certificates for: MYMACHINE.mydomain.com or the IP address.
|
||||
|
||||
2. Using a password is optional. If you choose to use a password, the same password applies to exported client and server certificates.
|
||||
a. Type a password in the Certificate password box.
|
||||
b. Repeat the password in the Retype password box.
|
||||
The passwords must match.
|
||||
|
||||
3. Select Include secret key if you want to add a secret key to the public key.
|
||||
|
||||
4. From the *Export file format for certificates* field select the "Platform independent PEM-format"
|
||||
|
||||
{% image
|
||||
src="/images/v1.1.1/connectors/qliksense/qlik-export-cert.png"
|
||||
alt="Provide Certificate Details"
|
||||
caption="Provide Certificate Details"
|
||||
/%}
|
||||
|
||||
|
||||
# Step 3: Locate the certificates
|
||||
|
||||
Once you have exported the certificates you can see the location of exported certificates just below the certificate details page. When you navigate to that location you will find the `root.pem`, `client.pem` & `client_key.pem` certificates which will be used by OpenMetadata.
|
||||
|
||||
{% image
|
||||
src="/images/v1.1.1/connectors/qliksense/qlik-locate-certificates.png"
|
||||
alt="Locate Certificate"
|
||||
caption="Locate Certificate"
|
||||
/%}
|
||||
@ -0,0 +1,98 @@
|
||||
---
|
||||
title: Qlik Sense
|
||||
slug: /connectors/dashboard/qliksense
|
||||
---
|
||||
|
||||
# Qlik Sense
|
||||
|
||||
| Stage | PROD |
|
||||
|------------|------------------------------|
|
||||
| Dashboards | {% icon iconName="check" /%} |
|
||||
| Charts | {% icon iconName="check" /%} |
|
||||
| Owners | {% icon iconName="cross" /%} |
|
||||
| Tags | {% icon iconName="cross" /%} |
|
||||
| Datamodels | {% icon iconName="check" /%} |
|
||||
| Lineage | {% icon iconName="check" /%} |
|
||||
|
||||
In this section, we provide guides and references to use the Qlik Sense connector.
|
||||
|
||||
Configure and schedule Metabase metadata and profiler workflows from the OpenMetadata UI:
|
||||
|
||||
- [Requirements](#requirements)
|
||||
- [Metadata Ingestion](#metadata-ingestion)
|
||||
|
||||
{% partial file="/v1.1.1/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/dashboard/qliksense/yaml"} /%}
|
||||
|
||||
## Requirements
|
||||
|
||||
{%inlineCallout icon="description" bold="OpenMetadata 1.1.1 or later" href="/deployment"%}
|
||||
To deploy OpenMetadata, check the Deployment guides.
|
||||
{%/inlineCallout%}
|
||||
|
||||
## Metadata Ingestion
|
||||
|
||||
{% partial
|
||||
file="/v1.1.1/connectors/metadata-ingestion-ui.md"
|
||||
variables={
|
||||
connector: "QlikSense",
|
||||
selectServicePath: "/images/v1.1.1/connectors/qliksense/select-service.png",
|
||||
addNewServicePath: "/images/v1.1.1/connectors/qliksense/add-new-service.png",
|
||||
serviceConnectionPath: "/images/v1.1.1/connectors/qliksense/service-connection.png",
|
||||
}
|
||||
/%}
|
||||
|
||||
{% stepsContainer %}
|
||||
{% extraContent parentTagName="stepsContainer" %}
|
||||
|
||||
#### Connection Details
|
||||
|
||||
- **Qlik Sense Base URL**: This field refers to the base url of your Qlik Sense Portal, will be used for generating the redirect links for dashboards and charts. Example: `https://server.domain.com` or `https://server.domain.com/<proxy-path>`
|
||||
- **Qlik Engine JSON API Websocket URL**: Enter the websocket url of Qlik Sense Engine JSON API. Refer to [this](https://help.qlik.com/en-US/sense-developer/May2023/Subsystems/EngineAPI/Content/Sense_EngineAPI/GettingStarted/connecting-to-engine-api.htm) document for more details about. Example: `wss://server.domain.com:4747` or `wss://server.domain.com[/virtual proxy]`.
|
||||
|
||||
Since we use the Qlik Sense Engine APIs, we need to authenticate to those APIs using certificates generated on Qlik Management Console.
|
||||
|
||||
**Qlik Certificate By Values**: In this approach we provide the content of the certificates to the relevant field.
|
||||
- **Client Certificate Value**: This field specifies the value of `client.pem` certificate required for authentication.
|
||||
- **Client Key Certificate Value**: This field specifies the value of `client_key.pem` certificate required for authentication.
|
||||
- **Root Certificate Value**: This field specifies the value of `root.pem` certificate required for authentication.
|
||||
- **Staging Directory Path**: This field specifies the path to temporary staging directory, where the certificates will be stored temporarily during the ingestion process, which will de deleted once the ingestion job is over.
|
||||
|
||||
when you are using this approach make sure you are passing the key in a correct format. If your certificate looks like this:
|
||||
|
||||
```
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MII..
|
||||
MBQ...
|
||||
CgU..
|
||||
8Lt..
|
||||
...
|
||||
h+4=
|
||||
-----END CERTIFICATE-----
|
||||
```
|
||||
|
||||
You will have to replace new lines with `\n` and the final private key that you need to pass should look like this:
|
||||
|
||||
```
|
||||
-----BEGIN CERTIFICATE-----\nMII..\nMBQ...\nCgU..\n8Lt..\n...\nh+4=\n-----END CERTIFICATE-----\n
|
||||
```
|
||||
|
||||
**Qlik Certificate By Path**: In this approach we provide the path of the certificates to the certificate stored in the container or environment running the ingestion workflow.
|
||||
- **Client Certificate Path**: This field specifies the path of `client.pem` certificate required for authentication.
|
||||
- **Client Key Certificate Value**: This field specifies the path of `client_key.pem` certificate required for authentication.
|
||||
- **Root Certificate Value**: This field specifies the path of `root.pem` certificate required for authentication.
|
||||
|
||||
**User Directory**: This field specifies the user directory of the user.
|
||||
|
||||
**User ID**: This field specifies the user id of the user.
|
||||
|
||||
{% /extraContent %}
|
||||
|
||||
{% partial file="/v1.1.1/connectors/test-connection.md" /%}
|
||||
|
||||
{% partial file="/v1.1.1/connectors/dashboard/configure-ingestion.md" /%}
|
||||
|
||||
{% partial file="/v1.1.1/connectors/ingestion-schedule-and-deploy.md" /%}
|
||||
|
||||
{% /stepsContainer %}
|
||||
|
||||
{% partial file="/v1.1.1/connectors/troubleshooting.md" /%}
|
||||
@ -0,0 +1,246 @@
|
||||
---
|
||||
title: Run the Qlik Sense Connector Externally
|
||||
slug: /connectors/dashboard/qliksense/yaml
|
||||
---
|
||||
|
||||
# Run the PowerBI Connector Externally
|
||||
|
||||
| Stage | PROD |
|
||||
|------------|------------------------------|
|
||||
| Dashboards | {% icon iconName="check" /%} |
|
||||
| Charts | {% icon iconName="check" /%} |
|
||||
| Owners | {% icon iconName="cross" /%} |
|
||||
| Tags | {% icon iconName="cross" /%} |
|
||||
| Datamodels | {% icon iconName="check" /%} |
|
||||
| Lineage | {% icon iconName="check" /%} |
|
||||
|
||||
In this section, we provide guides and references to use the PowerBI connector.
|
||||
|
||||
Configure and schedule PowerBI metadata and profiler workflows from the OpenMetadata UI:
|
||||
|
||||
- [Requirements](#requirements)
|
||||
- [Metadata Ingestion](#metadata-ingestion)
|
||||
|
||||
{% partial file="/v1.1.1/connectors/external-ingestion-deployment.md" /%}
|
||||
|
||||
## Requirements
|
||||
|
||||
{%inlineCallout icon="description" bold="OpenMetadata 0.12 or later" href="/deployment"%}
|
||||
To deploy OpenMetadata, check the Deployment guides.
|
||||
{%/inlineCallout%}
|
||||
|
||||
### Python Requirements
|
||||
|
||||
To run the PowerBI ingestion, you will need to install:
|
||||
|
||||
```bash
|
||||
pip3 install "openmetadata-ingestion[qliksense]"
|
||||
```
|
||||
|
||||
## Metadata Ingestion
|
||||
|
||||
All connectors are defined as JSON Schemas.
|
||||
[Here](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/dashboard/qlikSenseConnection.json)
|
||||
you can find the structure to create a connection to QlikSense.
|
||||
|
||||
In order to create and run a Metadata Ingestion workflow, we will follow
|
||||
the steps to create a YAML configuration able to connect to the source,
|
||||
process the Entities if needed, and reach the OpenMetadata server.
|
||||
|
||||
The workflow is modeled around the following
|
||||
[JSON Schema](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json)
|
||||
|
||||
### 1. Define the YAML Config
|
||||
|
||||
This is a sample config for Qlik Sense:
|
||||
|
||||
{% codePreview %}
|
||||
|
||||
{% codeInfoContainer %}
|
||||
|
||||
#### Source Configuration - Service Connection
|
||||
|
||||
{% codeInfo srNumber=1 %}
|
||||
|
||||
**hostPort**: Qlik Engine JSON API Websocket URL
|
||||
|
||||
Enter the websocket url of Qlik Sense Engine JSON API. Refer to [this](https://help.qlik.com/en-US/sense-developer/May2023/Subsystems/EngineAPI/Content/Sense_EngineAPI/GettingStarted/connecting-to-engine-api.htm) document for more details about
|
||||
|
||||
Example: `wss://server.domain.com:4747` or `wss://server.domain.com[/virtual proxy]`
|
||||
|
||||
**Note:** Notice that you have to provide the websocket url here which would begin with either `wss://` or `ws://`
|
||||
|
||||
{% /codeInfo %}
|
||||
|
||||
{% codeInfo srNumber=2 %}
|
||||
|
||||
**displayUrl**: Qlik Sense Base URL
|
||||
|
||||
This field refers to the base url of your Qlik Sense Portal, will be used for generating the redirect links for dashboards and charts.
|
||||
|
||||
Example: `https://server.domain.com` or `https://server.domain.com/<virtual-proxy-path>`
|
||||
|
||||
{% /codeInfo %}
|
||||
|
||||
{% codeInfo srNumber=3 %}
|
||||
|
||||
Since we use the Qlik Sense Engine APIs, we need to authenticate to those APIs using certificates generated on Qlik Management Console.
|
||||
|
||||
In this approach we provide the path of the certificates to the certificate stored in the container or environment running the ingestion workflow.
|
||||
|
||||
- **clientCertificate**: This field specifies the path of `client.pem` certificate required for authentication.
|
||||
- **clientKeyCertificate**: This field specifies the path of `client_key.pem` certificate required for authentication.
|
||||
- **rootCertificate**: This field specifies the path of `root.pem` certificate required for authentication.
|
||||
|
||||
{% /codeInfo %}
|
||||
|
||||
{% codeInfo srNumber=4 %}
|
||||
|
||||
In this approach we provide the content of the certificates to the relevant field.
|
||||
|
||||
- **Client Certificate Value**: This field specifies the value of `client.pem` certificate required for authentication.
|
||||
- **Client Key Certificate Value**: This field specifies the value of `client_key.pem` certificate required for authentication.
|
||||
- **Root Certificate Value**: This field specifies the value of `root.pem` certificate required for authentication.
|
||||
- **Staging Directory Path**: This field specifies the path to temporary staging directory, where the certificates will be stored temporarily during the ingestion process, which will de deleted once the ingestion job is over.
|
||||
|
||||
when you are using this approach make sure you are passing the key in a correct format. If your certificate looks like this:
|
||||
|
||||
```
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MII..
|
||||
MBQ...
|
||||
CgU..
|
||||
8Lt..
|
||||
...
|
||||
h+4=
|
||||
-----END CERTIFICATE-----
|
||||
```
|
||||
|
||||
You will have to replace new lines with `\n` and the final private key that you need to pass should look like this:
|
||||
|
||||
```
|
||||
-----BEGIN CERTIFICATE-----\nMII..\nMBQ...\nCgU..\n8Lt..\n...\nh+4=\n-----END CERTIFICATE-----\n
|
||||
```
|
||||
|
||||
|
||||
{% /codeInfo %}
|
||||
|
||||
{% codeInfo srNumber=5 %}
|
||||
|
||||
**userId**: This field specifies the user directory of the user.
|
||||
|
||||
{% /codeInfo %}
|
||||
|
||||
{% codeInfo srNumber=6 %}
|
||||
|
||||
**userDirectory**: This field specifies the user directory of the user.
|
||||
|
||||
{% /codeInfo %}
|
||||
|
||||
#### Source Configuration - Source Config
|
||||
|
||||
{% codeInfo srNumber=9 %}
|
||||
|
||||
The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json):
|
||||
|
||||
- **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
|
||||
- **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
|
||||
- **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
|
||||
- **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
|
||||
- **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
|
||||
- **markDeletedDashboards**: Set the 'Mark Deleted Dashboards' toggle to flag dashboards as soft-deleted if they are not present anymore in the source system.
|
||||
|
||||
{% /codeInfo %}
|
||||
|
||||
#### Sink Configuration
|
||||
|
||||
{% codeInfo srNumber=10 %}
|
||||
|
||||
To send the metadata to OpenMetadata, it needs to be specified as `type: metadata-rest`.
|
||||
|
||||
{% /codeInfo %}
|
||||
|
||||
{% partial file="/v1.1.1/connectors/workflow-config.md" /%}
|
||||
|
||||
{% /codeInfoContainer %}
|
||||
|
||||
{% codeBlock fileName="filename.yaml" %}
|
||||
|
||||
```yaml
|
||||
source:
|
||||
type: qliksense
|
||||
serviceName: local_qliksense
|
||||
serviceConnection:
|
||||
config:
|
||||
type: QlikSense
|
||||
```
|
||||
```yaml {% srNumber=1 %}
|
||||
hostPort: wss://localhost:4747
|
||||
```
|
||||
```yaml {% srNumber=2 %}
|
||||
displayUrl: https://localhost
|
||||
```
|
||||
```yaml {% srNumber=3 %}
|
||||
certificates:
|
||||
# pass certificate paths
|
||||
clientCertificate: /path/to/client.pem
|
||||
clientKeyCertificate: /path/to/client_key.pem
|
||||
rootCertificate: /path/to/root.pem
|
||||
```
|
||||
```yaml {% srNumber=4 %}
|
||||
# pass certificate values
|
||||
# clientCertificateData: -----BEGIN CERTIFICATE-----\n....\n.....\n-----END CERTIFICATE-----\n
|
||||
# clientKeyCertificateData: -----BEGIN RSA PRIVATE KEY-----\n....\n....\n-----END RSA PRIVATE KEY-----\n
|
||||
# rootCertificateData: -----BEGIN CERTIFICATE-----\n....\n...-----END CERTIFICATE-----\n
|
||||
# stagingDir: /tmp/stage
|
||||
```
|
||||
```yaml {% srNumber=5 %}
|
||||
userId: user_id
|
||||
```
|
||||
```yaml {% srNumber=6 %}
|
||||
userDirectory: user_dir
|
||||
```
|
||||
```yaml {% srNumber=7 %}
|
||||
sourceConfig:
|
||||
config:
|
||||
type: DashboardMetadata
|
||||
# dbServiceNames:
|
||||
# - service1
|
||||
# - service2
|
||||
# dashboardFilterPattern:
|
||||
# includes:
|
||||
# - dashboard1
|
||||
# - dashboard2
|
||||
# excludes:
|
||||
# - dashboard3
|
||||
# - dashboard4
|
||||
# chartFilterPattern:
|
||||
# includes:
|
||||
# - chart1
|
||||
# - chart2
|
||||
# excludes:
|
||||
# - chart3
|
||||
# - chart4
|
||||
```
|
||||
```yaml {% srNumber=8 %}
|
||||
sink:
|
||||
type: metadata-rest
|
||||
config: {}
|
||||
```
|
||||
|
||||
{% partial file="workflow-config-yaml.md" /%}
|
||||
|
||||
{% /codeBlock %}
|
||||
|
||||
{% /codePreview %}
|
||||
|
||||
### 2. Run with the CLI
|
||||
|
||||
First, we will need to save the YAML file. Afterward, and with all requirements installed, we can run:
|
||||
|
||||
```bash
|
||||
metadata ingest -c <path-to-yaml>
|
||||
```
|
||||
|
||||
Note that from connector to connector, this recipe will always be the same. By updating the YAML configuration,
|
||||
you will be able to extract metadata from different sources.
|
||||
@ -69,6 +69,8 @@ the following docs to run the Ingestion Framework in any orchestrator externally
|
||||
- [Superset](/connectors/dashboard/superset)
|
||||
- [Tableau](/connectors/dashboard/tableau)
|
||||
- [Domo Dashboard](/connectors/dashboard/domo-dashboard)
|
||||
- [Qlik Sense](/connectors/dashboard/qliksense)
|
||||
- [QuickSight](/connectors/dashboard/quicksight)
|
||||
|
||||
## Messaging Services
|
||||
|
||||
|
||||
@ -378,6 +378,12 @@ site_menu:
|
||||
url: /connectors/dashboard/quicksight
|
||||
- category: Connectors / Dashboard / QuickSight / Run Externally
|
||||
url: /connectors/dashboard/quicksight/yaml
|
||||
- category: Connectors / Dashboard / Qlik Sense
|
||||
url: /connectors/dashboard/qliksense
|
||||
- category: Connectors / Dashboard / Qlik Sense / Run Externally
|
||||
url: /connectors/dashboard/qliksense/yaml
|
||||
- category: Connectors / Dashboard / Qlik Sense / Export Certificates
|
||||
url: /connectors/dashboard/qliksense/certificates
|
||||
- category: Connectors / Dashboard / Redash
|
||||
url: /connectors/dashboard/redash
|
||||
- category: Connectors / Dashboard / Redash / Run Externally
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 117 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 306 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 104 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 49 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 171 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 349 KiB |
@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "QlikSense",
|
||||
"displayName": "QlikSense Test Connection",
|
||||
"description": "This Test Connection validates the access against the server and basic metadata extraction of dashboards and charts.",
|
||||
"steps": [
|
||||
{
|
||||
"name": "GetDashboards",
|
||||
"description": "List all the dashboards available to the user",
|
||||
"errorMessage": "Failed to fetch dashboards, please validate the credentials or validate if user has access to fetch dashboards",
|
||||
"shortCircuit": true,
|
||||
"mandatory": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -19,7 +19,8 @@
|
||||
"MetabaseDataModel",
|
||||
"LookMlView",
|
||||
"LookMlExplore",
|
||||
"PowerBIDataModel"
|
||||
"PowerBIDataModel",
|
||||
"QlikSenseDataModel"
|
||||
],
|
||||
"javaEnums": [
|
||||
{
|
||||
@ -39,6 +40,9 @@
|
||||
},
|
||||
{
|
||||
"name": "PowerBIDataModel"
|
||||
},
|
||||
{
|
||||
"name": "QlikSenseDataModel"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -0,0 +1,133 @@
|
||||
{
|
||||
"$id": "https://open-metadata.org/schema/entity/services/connections/dashboard/qlikSenseConnection.json",
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "QlikSenseConnection",
|
||||
"description": "Qlik Sense Connection Config",
|
||||
"type": "object",
|
||||
"javaType": "org.openmetadata.schema.services.connections.dashboard.QlikSenseConnection",
|
||||
"definitions": {
|
||||
"qlikSenseType": {
|
||||
"description": "Qlik sense service type",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"QlikSense"
|
||||
],
|
||||
"default": "QlikSense"
|
||||
},
|
||||
"qlikCertificatePath": {
|
||||
"description": "Qlik Authentication Certificate File Path",
|
||||
"title": "Qlik Certificates By File Path",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"clientCertificate": {
|
||||
"title": "Client Certificate Path",
|
||||
"description": "Client Certificate",
|
||||
"type": "string"
|
||||
},
|
||||
"clientKeyCertificate": {
|
||||
"title": "Client Key Certificate",
|
||||
"description": "Client Key Certificate.",
|
||||
"type": "string"
|
||||
},
|
||||
"rootCertificate": {
|
||||
"title": "Root Certificate",
|
||||
"description": "Root Certificate.",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"clientCertificate",
|
||||
"clientKeyCertificate",
|
||||
"rootCertificate"
|
||||
]
|
||||
},
|
||||
"qlikCertificateValues": {
|
||||
"description": "Qlik Authentication Certificate By Values",
|
||||
"title": "Qlik Certificates By Values",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"clientCertificateData": {
|
||||
"title": "Client Certificate Value",
|
||||
"description": "Client Certificate",
|
||||
"type": "string",
|
||||
"format": "password"
|
||||
},
|
||||
"clientKeyCertificateData": {
|
||||
"title": "Client Key Certificate Value",
|
||||
"description": "Client Key Certificate.",
|
||||
"type": "string",
|
||||
"format": "password"
|
||||
},
|
||||
"rootCertificateData": {
|
||||
"title": "Root Certificate Value",
|
||||
"description": "Root Certificate.",
|
||||
"type": "string",
|
||||
"format": "password"
|
||||
},
|
||||
"stagingDir": {
|
||||
"title": "Staging Directory Path",
|
||||
"description": "Staging Directory Path",
|
||||
"type": "string",
|
||||
"default": "/tmp/openmetadata-qlik"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"clientCertificateData",
|
||||
"clientKeyCertificateData",
|
||||
"rootCertificateData",
|
||||
"stagingDir"
|
||||
]
|
||||
}
|
||||
},
|
||||
"properties": {
|
||||
"type": {
|
||||
"title": "Service Type",
|
||||
"description": "Service Type",
|
||||
"$ref": "#/definitions/qlikSenseType",
|
||||
"default": "QlikSense"
|
||||
},
|
||||
"displayUrl": {
|
||||
"expose": true,
|
||||
"title": "Qlik Sense Base URL",
|
||||
"description": "Qlik Sense Base URL, used for genrating dashboard & chat url",
|
||||
"type": "string",
|
||||
"format": "uri"
|
||||
},
|
||||
"hostPort": {
|
||||
"expose": true,
|
||||
"title": "Qlik Engine JSON API Websocket URL",
|
||||
"description": "URL for the superset instance.",
|
||||
"type": "string",
|
||||
"format": "uri"
|
||||
},
|
||||
"certificates": {
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/definitions/qlikCertificateValues"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/qlikCertificatePath"
|
||||
}
|
||||
]
|
||||
},
|
||||
"userDirectory": {
|
||||
"title": "User Directory",
|
||||
"description": "User Directory.",
|
||||
"type": "string"
|
||||
},
|
||||
"userId": {
|
||||
"title": "User ID",
|
||||
"description": "User ID.",
|
||||
"type": "string"
|
||||
},
|
||||
"supportsMetadataExtraction": {
|
||||
"title": "Supports Metadata Extraction",
|
||||
"$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"hostPort",
|
||||
"certificates"
|
||||
]
|
||||
}
|
||||
@ -24,7 +24,8 @@
|
||||
"Mode",
|
||||
"CustomDashboard",
|
||||
"DomoDashboard",
|
||||
"QuickSight"
|
||||
"QuickSight",
|
||||
"QlikSense"
|
||||
],
|
||||
"javaEnums": [
|
||||
{
|
||||
@ -56,6 +57,9 @@
|
||||
},
|
||||
{
|
||||
"name": "QuickSight"
|
||||
},
|
||||
{
|
||||
"name": "QlikSense"
|
||||
}
|
||||
]
|
||||
},
|
||||
@ -99,6 +103,9 @@
|
||||
},
|
||||
{
|
||||
"$ref": "./connections/dashboard/quickSightConnection.json"
|
||||
},
|
||||
{
|
||||
"$ref": "./connections/dashboard/qlikSenseConnection.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -0,0 +1,156 @@
|
||||
# QlikSense
|
||||
|
||||
In this section, we provide guides and references to use the Metabase connector.
|
||||
|
||||
## Requirements
|
||||
|
||||
We will extract the metadata using the [Qlik Sense Engine JSON API](https://help.qlik.com/en-US/sense-developer/May2023/Subsystems/EngineAPI/Content/Sense_EngineAPI/introducing-engine-API.htm).
|
||||
|
||||
You can find further information on the Qlik Sense connector in the [docs](https://docs.open-metadata.org/connectors/dashboard/qliksense).
|
||||
|
||||
## Connection Details
|
||||
|
||||
$$section
|
||||
### Qlik Sense Base URL $(id="displayUrl")
|
||||
|
||||
This field refers to the base url of your Qlik Sense Portal, will be used for generating the redirect links for dashboards and charts.
|
||||
|
||||
Example: `https://server.domain.com` or `https://server.domain.com/<virtual-proxy-path>`
|
||||
$$
|
||||
|
||||
$$section
|
||||
### Qlik Engine JSON API Websocket URL $(id="hostPort")
|
||||
|
||||
Enter the websocket url of Qlik Sense Engine JSON API. Refer to [this](https://help.qlik.com/en-US/sense-developer/May2023/Subsystems/EngineAPI/Content/Sense_EngineAPI/GettingStarted/connecting-to-engine-api.htm) document for more details about
|
||||
|
||||
Example: `wss://server.domain.com:4747` or `wss://server.domain.com[/virtual proxy]`
|
||||
|
||||
**Note:** Notice that you have to provide the websocket url here which would begin with either `wss://` or `ws://`
|
||||
$$
|
||||
|
||||
$$section
|
||||
### Client Certificate Value $(id="clientCertificateData")
|
||||
|
||||
This field specifies the value of `client.pem` certificate required for authentication.
|
||||
|
||||
|
||||
Make sure you are passing the key in a correct format. If your certificate looks like this:
|
||||
|
||||
```
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MII..
|
||||
MBQ...
|
||||
CgU..
|
||||
8Lt..
|
||||
...
|
||||
h+4=
|
||||
-----END CERTIFICATE-----
|
||||
```
|
||||
|
||||
You will have to replace new lines with `\n` and the final private key that you need to pass should look like this:
|
||||
|
||||
```
|
||||
-----BEGIN CERTIFICATE-----\nMII..\nMBQ...\nCgU..\n8Lt..\n...\nh+4=\n-----END CERTIFICATE-----\n
|
||||
```
|
||||
$$
|
||||
|
||||
|
||||
$$section
|
||||
### Client Key Certificate Value $(id="clientKeyCertificateData")
|
||||
|
||||
This field specifies the value of `client_key.pem` certificate required for authentication.
|
||||
|
||||
|
||||
Make sure you are passing the key in a correct format. If your certificate looks like this:
|
||||
|
||||
```
|
||||
-----BEGIN RSA PRIVATE KEY-----
|
||||
MII..
|
||||
MBQ...
|
||||
CgU..
|
||||
8Lt..
|
||||
...
|
||||
h+4=
|
||||
-----END RSA PRIVATE KEY-----
|
||||
```
|
||||
|
||||
You will have to replace new lines with `\n` and the final private key that you need to pass should look like this:
|
||||
|
||||
```
|
||||
-----BEGIN CERTIFICATE-----\nMII..\nMBQ...\nCgU..\n8Lt..\n...\nh+4=\n-----END CERTIFICATE-----\n
|
||||
```
|
||||
$$
|
||||
|
||||
|
||||
$$section
|
||||
### Root Certificate Value $(id="rootCertificateData")
|
||||
|
||||
This field specifies the value of `root.pem` certificate required for authentication.
|
||||
|
||||
|
||||
Make sure you are passing the key in a correct format. If your certificate looks like this:
|
||||
|
||||
```
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MII..
|
||||
MBQ...
|
||||
CgU..
|
||||
8Lt..
|
||||
...
|
||||
h+4=
|
||||
-----END CERTIFICATE-----
|
||||
```
|
||||
|
||||
You will have to replace new lines with `\n` and the final private key that you need to pass should look like this:
|
||||
|
||||
```
|
||||
-----BEGIN CERTIFICATE-----\nMII..\nMBQ...\nCgU..\n8Lt..\n...\nh+4=\n-----END CERTIFICATE-----\n
|
||||
```
|
||||
$$
|
||||
|
||||
|
||||
$$section
|
||||
### Staging Directory Path $(id="stagingDir")
|
||||
|
||||
This field specifies the path to temporary staging directory, where the certificates will be stored temporarily during the ingestion process, which will de deleted once the ingestion job is over.
|
||||
$$
|
||||
|
||||
$$section
|
||||
### Client Certificate Path $(id="clientCertificate")
|
||||
|
||||
This field specifies the path of `client.pem` certificate required for authentication.
|
||||
|
||||
Example: `/path/to/client.pem`
|
||||
$$
|
||||
|
||||
|
||||
$$section
|
||||
### Client Key Certificate Path $(id="clientKeyCertificate")
|
||||
|
||||
This field specifies the path of `client_key.pem` certificate required for authentication.
|
||||
|
||||
Example: `/path/to/client_key.pem`
|
||||
$$
|
||||
|
||||
|
||||
$$section
|
||||
### Root Certificate Path $(id="rootCertificate")
|
||||
|
||||
This field specifies the path of `root.pem` certificate required for authentication.
|
||||
|
||||
Example: `/path/to/root.pem`
|
||||
$$
|
||||
|
||||
$$section
|
||||
### User Directory $(id="userDirectory")
|
||||
|
||||
This field specifies the user directory of the user.
|
||||
$$
|
||||
|
||||
|
||||
|
||||
$$section
|
||||
### User ID $(id="userId")
|
||||
|
||||
This field specifies the user id of the user.
|
||||
$$
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 75 KiB |
@ -59,6 +59,7 @@ import powerbi from '../assets/img/service-icon-power-bi.png';
|
||||
import prefect from '../assets/img/service-icon-prefect.png';
|
||||
import presto from '../assets/img/service-icon-presto.png';
|
||||
import pulsar from '../assets/img/service-icon-pulsar.png';
|
||||
import qlikSense from '../assets/img/service-icon-qlik-sense.png';
|
||||
import query from '../assets/img/service-icon-query.png';
|
||||
import quicksight from '../assets/img/service-icon-quicksight.png';
|
||||
import redash from '../assets/img/service-icon-redash.png';
|
||||
@ -171,6 +172,7 @@ export const GCS = gcs;
|
||||
export const MS_AZURE = msAzure;
|
||||
export const SPLINE = spline;
|
||||
export const MONGODB = mongodb;
|
||||
export const QLIK_SENSE = qlikSense;
|
||||
|
||||
export const PLUS = plus;
|
||||
export const NOSERVICE = noService;
|
||||
@ -350,6 +352,7 @@ export const BETA_SERVICES = [
|
||||
DatabaseServiceType.SapHana,
|
||||
PipelineServiceType.Spline,
|
||||
DatabaseServiceType.MongoDB,
|
||||
DashboardServiceType.QlikSense,
|
||||
];
|
||||
|
||||
export const TEST_CONNECTION_INITIAL_MESSAGE = i18n.t(
|
||||
|
||||
@ -23,6 +23,7 @@ import lookerConnection from '../jsons/connectionSchemas/connections/dashboard/l
|
||||
import metabaseConnection from '../jsons/connectionSchemas/connections/dashboard/metabaseConnection.json';
|
||||
import modeConnection from '../jsons/connectionSchemas/connections/dashboard/modeConnection.json';
|
||||
import powerBIConnection from '../jsons/connectionSchemas/connections/dashboard/powerBIConnection.json';
|
||||
import qliksenseConnection from '../jsons/connectionSchemas/connections/dashboard/qlikSenseConnection.json';
|
||||
import quicksightConnection from '../jsons/connectionSchemas/connections/dashboard/quickSightConnection.json';
|
||||
import redashConnection from '../jsons/connectionSchemas/connections/dashboard/redashConnection.json';
|
||||
import tableauConnection from '../jsons/connectionSchemas/connections/dashboard/tableauConnection.json';
|
||||
@ -87,6 +88,11 @@ export const getDashboardConfig = (type: DashboardServiceType) => {
|
||||
case DashboardServiceType.QuickSight: {
|
||||
schema = quicksightConnection;
|
||||
|
||||
break;
|
||||
}
|
||||
case DashboardServiceType.QlikSense: {
|
||||
schema = qliksenseConnection;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@ -67,6 +67,7 @@ import {
|
||||
POSTGRES,
|
||||
POWERBI,
|
||||
PRESTO,
|
||||
QLIK_SENSE,
|
||||
QUICKSIGHT,
|
||||
REDASH,
|
||||
REDPANDA,
|
||||
@ -245,6 +246,9 @@ export const serviceTypeLogo = (type: string) => {
|
||||
case DashboardServiceType.Mode:
|
||||
return MODE;
|
||||
|
||||
case DashboardServiceType.QlikSense:
|
||||
return QLIK_SENSE;
|
||||
|
||||
case PipelineServiceType.Airflow:
|
||||
return AIRFLOW;
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user