diff --git a/ingestion/setup.py b/ingestion/setup.py index 0ce66c4a623..87f654463be 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -213,6 +213,7 @@ plugins: Dict[str, Set[str]] = { VERSIONS["packaging"], }, "powerbi": {VERSIONS["msal"]}, + "qliksense": {"websockets~=11.0.3"}, "presto": {*COMMONS["hive"]}, "pymssql": {"pymssql==2.2.5"}, "quicksight": {VERSIONS["boto3"]}, diff --git a/ingestion/src/metadata/examples/workflows/qlik_sense.yaml b/ingestion/src/metadata/examples/workflows/qlik_sense.yaml new file mode 100644 index 00000000000..79cd9ae6690 --- /dev/null +++ b/ingestion/src/metadata/examples/workflows/qlik_sense.yaml @@ -0,0 +1,43 @@ +source: + type: qliksense + serviceName: final_lineage + serviceConnection: + config: + type: QlikSense + hostPort: wss://localhost:4747 + displayUrl: https://localhost + certificates: + # pass certificate paths + clientCertificate: /path/to/client.pem + clientKeyCertificate: /path/to/client_key.pem + rootCertificate: /path/to/root.pem + + # pass certificate values + # clientCertificateData: -----BEGIN CERTIFICATE-----\n....\n.....\n-----END CERTIFICATE-----\n + # clientKeyCertificateData: -----BEGIN RSA PRIVATE KEY-----\n....\n....\n-----END RSA PRIVATE KEY-----\n + # rootCertificateData: -----BEGIN CERTIFICATE-----\n....\n...-----END CERTIFICATE-----\n + # stagingDir: /tmp/stage + + userId: user_id + userDirectory: user_dir + sourceConfig: + config: + type: DashboardMetadata + dbServiceNames: + - mysql + - postgres + chartFilterPattern: {} + # dashboardFilterPattern: + # includes: + # - .*mysql.* + +sink: + type: metadata-rest + config: {} +workflowConfig: + loggerLevel: DEBUG + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" diff --git a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py index 6a6b2465bec..d4e26da679a 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py @@ -288,9 +288,28 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC): f"DataModel is not supported for {self.service_connection.type.name}" ) + def yield_datamodel_dashboard_lineage( + self, + ) -> Iterable[AddLineageRequest]: + """ + Returns: + Lineage request between Data Models and Dashboards + """ + if hasattr(self.context, "dataModels") and self.context.dataModels: + for datamodel in self.context.dataModels: + try: + yield self._get_add_lineage_request( + to_entity=self.context.dashboard, from_entity=datamodel + ) + except Exception as err: + logger.debug(traceback.format_exc()) + logger.error( + f"Error to yield dashboard lineage details for data model name [{datamodel.name}]: {err}" + ) + def yield_dashboard_lineage( self, dashboard_details: Any - ) -> Optional[Iterable[AddLineageRequest]]: + ) -> Iterable[AddLineageRequest]: """ Yields lineage if config is enabled. @@ -301,6 +320,8 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC): on the dbServiceNames since our lineage will now be model -> dashboard """ + yield from self.yield_datamodel_dashboard_lineage() or [] + for db_service_name in self.source_config.dbServiceNames or []: yield from self.yield_dashboard_lineage_details( dashboard_details, db_service_name diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qliksense/client.py b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/client.py new file mode 100644 index 00000000000..d3751fccc1a --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/client.py @@ -0,0 +1,223 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Websocket Auth & Client for QlikSense +""" +import json +import traceback +from pathlib import Path +from typing import Dict, List, Optional + +from pydantic import ValidationError + +from metadata.generated.schema.entity.services.connections.dashboard.qlikSenseConnection import ( + QlikCertificatePath, + QlikCertificateValues, + QlikSenseConnection, +) +from metadata.ingestion.source.dashboard.qliksense.constants import ( + APP_LOADMODEL_REQ, + CREATE_SHEET_SESSION, + GET_DOCS_LIST_REQ, + GET_LOADMODEL_LAYOUT, + GET_SHEET_LAYOUT, + OPEN_DOC_REQ, +) +from metadata.ingestion.source.dashboard.qliksense.models import ( + QlikDashboard, + QlikDashboardResult, + QlikDataModelResult, + QlikSheet, + QlikSheetResult, + QlikTable, +) +from metadata.utils.constants import UTF_8 +from metadata.utils.helpers import clean_uri, delete_dir_content, init_staging_dir +from metadata.utils.logger import ingestion_logger + +logger = ingestion_logger() + +QLIK_USER_HEADER = "X-Qlik-User" + + +class QlikSenseClient: + """ + Client Handling API communication with Qlik Engine APIs + """ + + def _clean_cert_value(self, cert_data: str) -> str: + return cert_data.replace("\\n", "\n") + + def write_data_to_file(self, file_path: Path, cert_data: str) -> None: + with open( + file_path, + "w+", + encoding=UTF_8, + ) as file: + data = self._clean_cert_value(cert_data) + + file.write(data) + + def _get_ssl_context(self) -> Optional[dict]: + if isinstance(self.config.certificates, QlikCertificatePath): + context = { + "ca_certs": self.config.certificates.rootCertificate, + "certfile": self.config.certificates.clientCertificate, + "keyfile": self.config.certificates.clientKeyCertificate, + } + return context + + init_staging_dir(self.config.certificates.stagingDir) + root_path = Path(self.config.certificates.stagingDir, "root.pem") + client_path = Path(self.config.certificates.stagingDir, "client.pem") + client_key_path = Path(self.config.certificates.stagingDir, "client_key.pem") + + self.write_data_to_file( + root_path, self.config.certificates.rootCertificateData.get_secret_value() + ) + + self.write_data_to_file( + client_path, + self.config.certificates.clientCertificateData.get_secret_value(), + ) + + self.write_data_to_file( + client_key_path, + self.config.certificates.clientKeyCertificateData.get_secret_value(), + ) + + context = { + "ca_certs": root_path, + "certfile": client_path, + "keyfile": client_key_path, + } + + return context + + def connect_websocket(self, app_id: str = None) -> None: + """ + Method to initialise websocket connection + """ + import ssl + + from websocket import create_connection + + if self.socket_connection: + self.socket_connection.close() + + ssl_conext = self._get_ssl_context() + ssl.match_hostname = lambda cert, hostname: True + self.socket_connection = create_connection( + f"{clean_uri(self.config.hostPort)}/app/{app_id or ''}", + sslopt=ssl_conext, + header={ + f"{QLIK_USER_HEADER}: " + f"UserDirectory={self.config.userDirectory}; UserId={self.config.userId}" + }, + ) + if app_id: + # get doc list needs to be executed before extracting data from app + self.get_dashboards_list(create_new_socket=False) + + def close_websocket(self) -> None: + if self.socket_connection: + self.socket_connection.close() + + if isinstance(self.config.certificates, QlikCertificateValues): + delete_dir_content(self.config.certificates.stagingDir) + + def __init__( + self, + config: QlikSenseConnection, + ) -> None: + self.config = config + self.socket_connection = None + + def _websocket_send_request( + self, request: dict, response: bool = False + ) -> Optional[Dict]: + """ + Method to send request to websocket + + request: data required to be sent to websocket + response: is json response required? + """ + self.socket_connection.send(json.dumps(request)) + resp = self.socket_connection.recv() + if response: + return json.loads(resp) + return None + + def get_dashboards_list( + self, create_new_socket: bool = True + ) -> List[QlikDashboard]: + """ + Get List of all dashboards + """ + try: + if create_new_socket: + self.connect_websocket() + self._websocket_send_request(GET_DOCS_LIST_REQ) + resp = self.socket_connection.recv() + dashboard_result = QlikDashboardResult(**json.loads(resp)) + return dashboard_result.result.qDocList + except Exception: + logger.debug(traceback.format_exc()) + logger.warning("Failed to fetch the dashboard list") + return [] + + def get_dashboard_charts(self, dashboard_id: str) -> List[QlikSheet]: + """ + Get dahsboard chart list + """ + try: + OPEN_DOC_REQ.update({"params": [dashboard_id]}) + self._websocket_send_request(OPEN_DOC_REQ) + self._websocket_send_request(CREATE_SHEET_SESSION) + sheets = self._websocket_send_request(GET_SHEET_LAYOUT, response=True) + data = QlikSheetResult(**sheets) + return data.result.qLayout.qAppObjectList.qItems + except Exception: + logger.debug(traceback.format_exc()) + logger.warning("Failed to fetch the dashboard charts") + return [] + + def get_dashboard_models(self) -> List[QlikTable]: + """ + Get dahsboard chart list + """ + try: + self._websocket_send_request(APP_LOADMODEL_REQ) + models = self._websocket_send_request(GET_LOADMODEL_LAYOUT, response=True) + data_models = QlikDataModelResult(**models) + layout = data_models.result.qLayout + if isinstance(layout, list): + tables = [] + for layout in data_models.result.qLayout: + tables.extend(layout.value.tables) + return tables + return layout.tables + except Exception: + logger.debug(traceback.format_exc()) + logger.warning("Failed to fetch the dashboard datamodels") + return [] + + def get_dashboard_for_test_connection(self): + try: + self.connect_websocket() + self._websocket_send_request(GET_DOCS_LIST_REQ) + resp = self.socket_connection.recv() + self.close_websocket() + return QlikDashboardResult(**json.loads(resp)) + except ValidationError: + logger.debug(traceback.format_exc()) + logger.warning("Failed to fetch the dashboard datamodels") + return None diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qliksense/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/connection.py new file mode 100644 index 00000000000..94ed95e10d5 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/connection.py @@ -0,0 +1,53 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Source connection handler +""" +from typing import Optional + +from metadata.generated.schema.entity.automations.workflow import ( + Workflow as AutomationWorkflow, +) +from metadata.generated.schema.entity.services.connections.dashboard.qlikSenseConnection import ( + QlikSenseConnection, +) +from metadata.ingestion.connections.test_connections import test_connection_steps +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.dashboard.qliksense.client import QlikSenseClient + + +def get_connection(connection: QlikSenseConnection) -> QlikSenseClient: + """ + Create connection + """ + return QlikSenseClient(connection) + + +def test_connection( + metadata: OpenMetadata, + client: QlikSenseClient, + service_connection: QlikSenseConnection, + automation_workflow: Optional[AutomationWorkflow] = None, +) -> None: + """ + Test connection. This can be executed either as part + of a metadata workflow or during an Automation Workflow + """ + + test_fn = {"GetDashboards": client.get_dashboard_for_test_connection} + + test_connection_steps( + metadata=metadata, + test_fn=test_fn, + service_type=service_connection.type.value, + automation_workflow=automation_workflow, + ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qliksense/constants.py b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/constants.py new file mode 100644 index 00000000000..1e726facc7e --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/constants.py @@ -0,0 +1,80 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +QlikSense Constants +""" + + +GET_DOCS_LIST_REQ = { + "handle": -1, + "method": "GetDocList", + "params": [], + "outKey": -1, + "id": 1, +} + +OPEN_DOC_REQ = { + "method": "OpenDoc", + "handle": -1, + "outKey": -1, + "id": 2, +} + +CREATE_SHEET_SESSION = { + "method": "CreateSessionObject", + "handle": 1, + "params": [ + { + "qInfo": {"qType": "SheetList"}, + "qAppObjectListDef": { + "qType": "sheet", + "qData": { + "title": "/qMetaDef/title", + "description": "/qMetaDef/description", + "thumbnail": "/thumbnail", + "cells": "/cells", + "rank": "/rank", + "columns": "/columns", + "rows": "/rows", + }, + }, + } + ], + "outKey": -1, + "id": 3, +} + +GET_SHEET_LAYOUT = { + "method": "GetLayout", + "handle": 2, + "params": [], + "outKey": -1, + "id": 4, +} + +APP_LOADMODEL_REQ = { + "delta": True, + "handle": 1, + "method": "GetObject", + "params": ["LoadModel"], + "id": 5, + "jsonrpc": "2.0", +} + + +GET_LOADMODEL_LAYOUT = { + "delta": True, + "handle": 3, + "method": "GetLayout", + "params": [], + "id": 6, + "jsonrpc": "2.0", +} diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qliksense/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/metadata.py new file mode 100644 index 00000000000..66d43aeb86e --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/metadata.py @@ -0,0 +1,327 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Qlik Sense Source Module""" + +import traceback +from typing import Iterable, List, Optional + +from metadata.generated.schema.api.data.createChart import CreateChartRequest +from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest +from metadata.generated.schema.api.data.createDashboardDataModel import ( + CreateDashboardDataModelRequest, +) +from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest +from metadata.generated.schema.entity.data.chart import Chart, ChartType +from metadata.generated.schema.entity.data.dashboardDataModel import ( + DashboardDataModel, + DataModelType, +) +from metadata.generated.schema.entity.data.table import Column, DataType, Table +from metadata.generated.schema.entity.services.connections.dashboard.qlikSenseConnection import ( + QlikSenseConnection, +) +from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( + OpenMetadataConnection, +) +from metadata.generated.schema.entity.services.dashboardService import ( + DashboardServiceType, +) +from metadata.generated.schema.entity.services.databaseService import DatabaseService +from metadata.generated.schema.metadataIngestion.workflow import ( + Source as WorkflowSource, +) +from metadata.ingestion.api.source import InvalidSourceException +from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource +from metadata.ingestion.source.dashboard.qliksense.client import QlikSenseClient +from metadata.ingestion.source.dashboard.qliksense.models import ( + QlikDashboard, + QlikTable, +) +from metadata.utils import fqn +from metadata.utils.filters import filter_by_chart, filter_by_datamodel +from metadata.utils.helpers import clean_uri +from metadata.utils.logger import ingestion_logger + +logger = ingestion_logger() + + +class QliksenseSource(DashboardServiceSource): + """ + Qlik Sense Source Class + """ + + config: WorkflowSource + client: QlikSenseClient + metadata_config: OpenMetadataConnection + + @classmethod + def create(cls, config_dict, metadata_config: OpenMetadataConnection): + config = WorkflowSource.parse_obj(config_dict) + connection: QlikSenseConnection = config.serviceConnection.__root__.config + if not isinstance(connection, QlikSenseConnection): + raise InvalidSourceException( + f"Expected QlikSenseConnection, but got {connection}" + ) + return cls(config, metadata_config) + + def __init__( + self, + config: WorkflowSource, + metadata_config: OpenMetadataConnection, + ): + super().__init__(config, metadata_config) + self.collections: List[QlikDashboard] = [] + + def get_dashboards_list(self) -> Iterable[QlikDashboard]: + """ + Get List of all dashboards + """ + for dashboard in self.client.get_dashboards_list(): + # create app specific websocket + self.client.connect_websocket(dashboard.qDocId) + # clean data models for next iteration + self.data_models = [] + yield dashboard + + def get_dashboard_name(self, dashboard: QlikDashboard) -> str: + """ + Get Dashboard Name + """ + return dashboard.qDocName + + def get_dashboard_details(self, dashboard: QlikDashboard) -> dict: + """ + Get Dashboard Details + """ + return dashboard + + def yield_dashboard( + self, dashboard_details: QlikDashboard + ) -> Iterable[CreateDashboardRequest]: + """ + Method to Get Dashboard Entity + """ + try: + if self.service_connection.displayUrl: + dashboard_url = ( + f"{clean_uri(self.service_connection.displayUrl)}/sense/app/" + f"{dashboard_details.qDocId}/overview" + ) + else: + dashboard_url = None + + dashboard_request = CreateDashboardRequest( + name=dashboard_details.qDocId, + sourceUrl=dashboard_url, + displayName=dashboard_details.qDocName, + description=dashboard_details.qMeta.description, + charts=[ + fqn.build( + self.metadata, + entity_type=Chart, + service_name=self.context.dashboard_service.fullyQualifiedName.__root__, + chart_name=chart.name.__root__, + ) + for chart in self.context.charts + ], + service=self.context.dashboard_service.fullyQualifiedName.__root__, + ) + yield dashboard_request + self.register_record(dashboard_request=dashboard_request) + except Exception as exc: # pylint: disable=broad-except + logger.debug(traceback.format_exc()) + logger.warning( + f"Error creating dashboard [{dashboard_details.qDocName}]: {exc}" + ) + + def yield_dashboard_chart( + self, dashboard_details: QlikDashboard + ) -> Iterable[CreateChartRequest]: + """Get chart method + + Args: + dashboard_details: + Returns: + Iterable[CreateChartRequest] + """ + charts = self.client.get_dashboard_charts(dashboard_id=dashboard_details.qDocId) + for chart in charts: + try: + if not chart.qInfo.qId: + continue + if self.service_connection.displayUrl: + chart_url = ( + f"{clean_uri(self.service_connection.displayUrl)}/sense/app/{dashboard_details.qDocId}" + f"/sheet/{chart.qInfo.qId}" + ) + else: + chart_url = None + if chart.qMeta.title and filter_by_chart( + self.source_config.chartFilterPattern, chart.qMeta.title + ): + self.status.filter(chart.qMeta.title, "Chart Pattern not allowed") + continue + yield CreateChartRequest( + name=chart.qInfo.qId, + displayName=chart.qMeta.title, + description=chart.qMeta.description, + chartType=ChartType.Other, + sourceUrl=chart_url, + service=self.context.dashboard_service.fullyQualifiedName.__root__, + ) + self.status.scanned(chart.qMeta.title) + except Exception as exc: # pylint: disable=broad-except + logger.debug(traceback.format_exc()) + logger.warning(f"Error creating chart [{chart}]: {exc}") + + def get_column_info(self, data_source: QlikTable) -> Optional[List[Column]]: + """ + Args: + data_source: DataSource + Returns: + Columns details for Data Model + """ + datasource_columns = [] + for field in data_source.fields or []: + try: + parsed_fields = { + "dataTypeDisplay": "Qlik Field", + "dataType": DataType.UNKNOWN, + "name": field.id, + "displayName": field.name if field.name else field.id, + } + datasource_columns.append(Column(**parsed_fields)) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Error to yield datamodel column: {exc}") + return datasource_columns + + def yield_datamodel(self, dashboard_details: QlikDashboard): + if self.source_config.includeDataModels: + self.data_models = self.client.get_dashboard_models() + for data_model in self.data_models or []: + try: + data_model_name = ( + data_model.tableName if data_model.tableName else data_model.id + ) + if filter_by_datamodel( + self.source_config.dataModelFilterPattern, data_model_name + ): + self.status.filter(data_model_name, "Data model filtered out.") + continue + + data_model_request = CreateDashboardDataModelRequest( + name=data_model.id, + displayName=data_model_name, + service=self.context.dashboard_service.fullyQualifiedName.__root__, + dataModelType=DataModelType.QlikSenseDataModel.value, + serviceType=DashboardServiceType.QlikSense.value, + columns=self.get_column_info(data_model), + ) + yield data_model_request + self.status.scanned( + f"Data Model Scanned: {data_model_request.displayName}" + ) + except Exception as exc: + error_msg = f"Error yielding Data Model [{data_model_name}]: {exc}" + self.status.failed( + name=data_model_name, + error=error_msg, + stack_trace=traceback.format_exc(), + ) + logger.error(error_msg) + logger.debug(traceback.format_exc()) + + def _get_datamodel(self, datamodel: QlikTable): + datamodel_fqn = fqn.build( + self.metadata, + entity_type=DashboardDataModel, + service_name=self.context.dashboard_service.fullyQualifiedName.__root__, + data_model_name=datamodel.id, + ) + if datamodel_fqn: + return self.metadata.get_by_name( + entity=DashboardDataModel, + fqn=datamodel_fqn, + ) + return None + + def _get_database_table( + self, db_service_entity: DatabaseService, datamodel: QlikTable + ) -> Optional[Table]: + """ + Get the table entity for lineage + """ + # table.name in tableau can come as db.schema.table_name. Hence the logic to split it + if datamodel.tableName and db_service_entity: + try: + if len(datamodel.connectorProperties.tableQualifiers) > 1: + ( + database_name, + schema_name, + ) = datamodel.connectorProperties.tableQualifiers[-2:] + elif len(datamodel.connectorProperties.tableQualifiers) == 1: + schema_name = datamodel.connectorProperties.tableQualifiers[-1] + database_name = None + else: + schema_name, database_name = None, None + + table_fqn = fqn.build( + self.metadata, + entity_type=Table, + service_name=db_service_entity.name.__root__, + schema_name=schema_name, + table_name=datamodel.tableName, + database_name=database_name, + ) + if table_fqn: + return self.metadata.get_by_name( + entity=Table, + fqn=table_fqn, + ) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Error occured while finding table fqn: {exc}") + return None + + def yield_dashboard_lineage_details( + self, + dashboard_details: QlikDashboard, + db_service_name: Optional[str], + ) -> Iterable[AddLineageRequest]: + """Get lineage method + + Args: + dashboard_details + """ + db_service_entity = self.metadata.get_by_name( + entity=DatabaseService, fqn=db_service_name + ) + for datamodel in self.data_models or []: + try: + data_model_entity = self._get_datamodel(datamodel=datamodel) + if data_model_entity: + om_table = self._get_database_table( + db_service_entity, datamodel=datamodel + ) + if om_table: + yield self._get_add_lineage_request( + to_entity=data_model_entity, from_entity=om_table + ) + except Exception as err: + logger.debug(traceback.format_exc()) + logger.error( + f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {err}" + ) + + def close(self): + self.client.close_websocket() + return super().close() diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qliksense/models.py b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/models.py new file mode 100644 index 00000000000..76eb18ca1f6 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/models.py @@ -0,0 +1,119 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +QlikSense Models +""" +from typing import List, Optional, Union + +from pydantic import BaseModel + +# dashboard models + + +class QlikDashboardMeta(BaseModel): + description: Optional[str] + + +class QlikDashboard(BaseModel): + qDocName: str + qDocId: str + qTitle: str + qMeta: Optional[QlikDashboardMeta] = QlikDashboardMeta() + + +class QlikDashboardList(BaseModel): + qDocList: Optional[List[QlikDashboard]] = [] + + +class QlikDashboardResult(BaseModel): + result: Optional[QlikDashboardList] = QlikDashboardList() + + +# sheet models +class QlikSheetInfo(BaseModel): + qId: str + + +class QlikSheetMeta(BaseModel): + title: Optional[str] + description: Optional[str] + + +class QlikSheet(BaseModel): + qInfo: QlikSheetInfo + qMeta: Optional[QlikSheetMeta] = QlikSheetMeta() + + +class QlikSheetItems(BaseModel): + qItems: Optional[List[QlikSheet]] = [] + + +class QlikSheetAppObject(BaseModel): + qAppObjectList: Optional[QlikSheetItems] = QlikSheetItems() + + +class QlikSheetLayout(BaseModel): + qLayout: Optional[QlikSheetAppObject] = QlikSheetAppObject() + + +class QlikSheetResult(BaseModel): + result: Optional[QlikSheetLayout] = QlikSheetLayout() + + +# datamodel models +class QlikFields(BaseModel): + name: Optional[str] + id: Optional[str] + + +class QlikTableConnectionProp(BaseModel): + tableQualifiers: Optional[List[str]] = [] + + +class QlikTable(BaseModel): + tableName: Optional[str] + id: Optional[str] + connectorProperties: Optional[QlikTableConnectionProp] = QlikTableConnectionProp() + fields: Optional[List[QlikFields]] = [] + + +class QlikTablesList(BaseModel): + tables: Optional[List[QlikTable]] = [] + + +class QlikDataModelValue(BaseModel): + value: Optional[QlikTablesList] = QlikTablesList() + + +class QlikDataModelLayout(BaseModel): + qLayout: Optional[ + Union[QlikTablesList, List[QlikDataModelValue]] + ] = QlikTablesList() + + +class QlikDataModelResult(BaseModel): + result: Optional[QlikDataModelLayout] = QlikDataModelLayout() + + +class QlikLayoutHandle(BaseModel): + qHandle: Optional[int] = 2 + + +class QlikLayoutValue(BaseModel): + value: Optional[QlikLayoutHandle] = QlikLayoutHandle() + + +class QlikQReturn(BaseModel): + qReturn: Optional[Union[QlikLayoutHandle, List[QlikLayoutValue]]] = [] + + +class QlikLayoutResult(BaseModel): + result: Optional[QlikQReturn] = QlikQReturn() diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py index c76a4f1e4d6..341f4537eae 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py @@ -224,31 +224,3 @@ class SupersetSourceMixin(DashboardServiceSource): logger.debug(traceback.format_exc()) logger.warning(f"Error to yield datamodel column: {exc}") return datasource_columns - - def yield_dashboard_lineage( - self, dashboard_details: Union[FetchDashboard, DashboradResult] - ) -> Optional[Iterable[AddLineageRequest]]: - yield from self.yield_datamodel_dashboard_lineage() or [] - - for db_service_name in self.source_config.dbServiceNames or []: - yield from self.yield_dashboard_lineage_details( - dashboard_details, db_service_name - ) or [] - - def yield_datamodel_dashboard_lineage( - self, - ) -> Optional[Iterable[AddLineageRequest]]: - """ - Returns: - Lineage request between Data Models and Dashboards - """ - for datamodel in self.context.dataModels or []: - try: - yield self._get_add_lineage_request( - to_entity=self.context.dashboard, from_entity=datamodel - ) - except Exception as err: - logger.debug(traceback.format_exc()) - logger.error( - f"Error to yield dashboard lineage details for data model name [{datamodel.name}]: {err}" - ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py index ef4f9fe39a3..9edbc342944 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py @@ -266,34 +266,6 @@ class TableauSource(DashboardServiceSource): logger.debug(traceback.format_exc()) logger.warning(f"Error to yield dashboard for {dashboard_details}: {exc}") - def yield_dashboard_lineage( - self, dashboard_details: TableauDashboard - ) -> Optional[Iterable[AddLineageRequest]]: - yield from self.yield_datamodel_dashboard_lineage() or [] - - for db_service_name in self.source_config.dbServiceNames or []: - yield from self.yield_dashboard_lineage_details( - dashboard_details, db_service_name - ) or [] - - def yield_datamodel_dashboard_lineage( - self, - ) -> Optional[Iterable[AddLineageRequest]]: - """ - Returns: - Lineage request between Data Models and Dashboards - """ - for datamodel in self.context.dataModels or []: - try: - yield self._get_add_lineage_request( - to_entity=self.context.dashboard, from_entity=datamodel - ) - except Exception as err: - logger.debug(traceback.format_exc()) - logger.error( - f"Error to yield dashboard lineage details for data model name [{datamodel.name}]: {err}" - ) - def yield_dashboard_lineage_details( self, dashboard_details: TableauDashboard, db_service_name: str ) -> Optional[Iterable[AddLineageRequest]]: diff --git a/ingestion/src/metadata/ingestion/stage/table_usage.py b/ingestion/src/metadata/ingestion/stage/table_usage.py index 2ea9c8563c5..cc58a659b5f 100644 --- a/ingestion/src/metadata/ingestion/stage/table_usage.py +++ b/ingestion/src/metadata/ingestion/stage/table_usage.py @@ -30,6 +30,7 @@ from metadata.generated.schema.type.tableUsageCount import TableUsageCount from metadata.ingestion.api.stage import Stage from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.utils.constants import UTF_8 +from metadata.utils.helpers import init_staging_dir from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -60,9 +61,7 @@ class TableUsageStage(Stage[QueryParserData]): self.metadata = OpenMetadata(self.metadata_config) self.table_usage = {} self.table_queries = {} - - self.init_location() - + init_staging_dir(self.config.filename) self.wrote_something = False @classmethod diff --git a/ingestion/src/metadata/utils/helpers.py b/ingestion/src/metadata/utils/helpers.py index 882a68f9bcd..a74bdfe3bfd 100644 --- a/ingestion/src/metadata/utils/helpers.py +++ b/ingestion/src/metadata/utils/helpers.py @@ -17,10 +17,12 @@ from __future__ import annotations import itertools import re +import shutil import sys from datetime import datetime, timedelta from functools import wraps from math import floor, log +from pathlib import Path from time import perf_counter from typing import Any, Dict, Iterable, List, Optional, Tuple, Union @@ -455,3 +457,20 @@ def get_database_name_for_lineage( db_service_entity.connection.config.__dict__.get("databaseName") or DEFAULT_DATABASE ) + + +def delete_dir_content(directory: str) -> None: + location = Path(directory) + if location.is_dir(): + logger.info("Location exists, cleaning it up") + shutil.rmtree(directory) + + +def init_staging_dir(directory: str) -> None: + """ + Prepare the the staging directory + """ + delete_dir_content(directory=directory) + location = Path(directory) + logger.info(f"Creating the directory to store staging data in {location}") + location.mkdir(parents=True, exist_ok=True) diff --git a/ingestion/tests/unit/topology/dashboard/test_qliksense.py b/ingestion/tests/unit/topology/dashboard/test_qliksense.py new file mode 100644 index 00000000000..4bab3c8f7bf --- /dev/null +++ b/ingestion/tests/unit/topology/dashboard/test_qliksense.py @@ -0,0 +1,186 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Test QlikSense using the topology +""" + +from unittest import TestCase +from unittest.mock import patch + +import pytest + +from metadata.generated.schema.api.data.createChart import CreateChartRequest +from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest +from metadata.generated.schema.entity.services.dashboardService import ( + DashboardConnection, + DashboardService, + DashboardServiceType, +) +from metadata.generated.schema.metadataIngestion.workflow import ( + OpenMetadataWorkflowConfig, +) +from metadata.generated.schema.type.basic import FullyQualifiedEntityName +from metadata.ingestion.source.dashboard.qliksense.client import QlikSenseClient +from metadata.ingestion.source.dashboard.qliksense.metadata import QliksenseSource +from metadata.ingestion.source.dashboard.qliksense.models import ( + QlikDashboard, + QlikSheet, + QlikSheetInfo, + QlikSheetMeta, +) + +MOCK_DASHBOARD_SERVICE = DashboardService( + id="c3eb265f-5445-4ad3-ba5e-797d3a3071bb", + name="qliksense_source_test", + fullyQualifiedName=FullyQualifiedEntityName(__root__="qliksense_source_test"), + connection=DashboardConnection(), + serviceType=DashboardServiceType.QlikSense, +) + + +mock_qliksense_config = { + "source": { + "type": "qliksense", + "serviceName": "local_qliksensem", + "serviceConnection": { + "config": { + "type": "QlikSense", + "certificates": { + "rootCertificate": "/test/path/root.pem", + "clientKeyCertificate": "/test/path/client_key.pem", + "clientCertificate": "/test/path/client.pem", + }, + "userDirectory": "demo", + "userId": "demo", + "hostPort": "wss://test:4747", + "displayUrl": "https://test", + } + }, + "sourceConfig": { + "config": {"dashboardFilterPattern": {}, "chartFilterPattern": {}} + }, + }, + "sink": {"type": "metadata-rest", "config": {}}, + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": { + "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" + }, + } + }, +} +MOCK_DASHBOARD_NAME = "New Dashboard" + +MOCK_DASHBOARD_DETAILS = QlikDashboard( + qDocName=MOCK_DASHBOARD_NAME, + qDocId="1", + qTitle=MOCK_DASHBOARD_NAME, +) + +MOCK_CHARTS = [ + QlikSheet( + qInfo=QlikSheetInfo(qId="11"), qMeta=QlikSheetMeta(title="Top Salespeople") + ), + QlikSheet( + qInfo=QlikSheetInfo(qId="12"), + qMeta=QlikSheetMeta(title="Milan Datasets", description="dummy"), + ), +] + +EXPECTED_DASHBOARD = CreateDashboardRequest( + name="1", + displayName="New Dashboard", + sourceUrl="https://test/sense/app/1/overview", + charts=[], + tags=None, + owner=None, + service="qliksense_source_test", + extension=None, +) + +EXPECTED_DASHBOARDS = [ + CreateChartRequest( + name="11", + displayName="Top Salespeople", + chartType="Other", + sourceUrl="https://test/sense/app/1/sheet/11", + tags=None, + owner=None, + service="qliksense_source_test", + ), + CreateChartRequest( + name="12", + displayName="Milan Datasets", + chartType="Other", + sourceUrl="https://test/sense/app/1/sheet/12", + tags=None, + owner=None, + service="qliksense_source_test", + description="dummy", + ), +] + + +class QlikSenseUnitTest(TestCase): + """ + Implements the necessary methods to extract + QlikSense Unit Test + """ + + def __init__(self, methodName) -> None: + with patch.object( + QlikSenseClient, "get_dashboard_for_test_connection", return_value=None + ): + super().__init__(methodName) + # test_connection.return_value = False + self.config = OpenMetadataWorkflowConfig.parse_obj(mock_qliksense_config) + self.qliksense = QliksenseSource.create( + mock_qliksense_config["source"], + self.config.workflowConfig.openMetadataServerConfig, + ) + self.qliksense.context.__dict__[ + "dashboard_service" + ] = MOCK_DASHBOARD_SERVICE + + @pytest.mark.order(1) + def test_dashboard(self): + dashboard_list = [] + results = self.qliksense.yield_dashboard(MOCK_DASHBOARD_DETAILS) + for result in results: + if isinstance(result, CreateDashboardRequest): + dashboard_list.append(result) + self.assertEqual(EXPECTED_DASHBOARD, dashboard_list[0]) + + @pytest.mark.order(2) + def test_dashboard_name(self): + assert ( + self.qliksense.get_dashboard_name(MOCK_DASHBOARD_DETAILS) + == MOCK_DASHBOARD_NAME + ) + + @pytest.mark.order(3) + def test_chart(self): + dashboard_details = MOCK_DASHBOARD_DETAILS + with patch.object( + QlikSenseClient, "get_dashboard_charts", return_value=MOCK_CHARTS + ): + results = list(self.qliksense.yield_dashboard_chart(dashboard_details)) + chart_list = [] + for result in results: + if isinstance(result, CreateChartRequest): + chart_list.append(result) + for _, (expected, original) in enumerate( + zip(EXPECTED_DASHBOARDS, chart_list) + ): + self.assertEqual(expected, original) diff --git a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/index.md b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/index.md index 4ef7ddcd4ee..4f78309d328 100644 --- a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/index.md +++ b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/index.md @@ -16,6 +16,7 @@ This is the supported list of connectors for Dashboard Services: - [Redash](/connectors/dashboard/redash) - [Superset](/connectors/dashboard/superset) - [Tableau](/connectors/dashboard/tableau) +- [Qlik Sense](/connectors/dashboard/qliksense) If you have a request for a new connector, don't hesitate to reach out in [Slack](https://slack.open-metadata.org/) or open a [feature request](https://github.com/open-metadata/OpenMetadata/issues/new/choose) in our GitHub repo. diff --git a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/qliksense/certificates.md b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/qliksense/certificates.md new file mode 100644 index 00000000000..0020a36f29d --- /dev/null +++ b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/qliksense/certificates.md @@ -0,0 +1,52 @@ +--- +title: Qlik Sense +slug: /connectors/dashboard/qliksense/certificates +--- + +# How to generate authentication certificates + +OpenMetadata Uses [Qlik Engine APIs](https://help.qlik.com/en-US/sense-developer/May2023/Subsystems/EngineAPI/Content/Sense_EngineAPI/introducing-engine-API.htm) to communicate with Qlik Sense and fetch relevant metadata, and connecting to these APIs require authentication certificates as described in [these docs](https://help.qlik.com/en-US/sense-developer/May2023/Subsystems/EngineAPI/Content/Sense_EngineAPI/GettingStarted/connecting-to-engine-api.htm). + + +In this document we will explain how you can generate these certificates so that OpenMetadata can communicate with Qlik Sense. + + +# Step 1: Open Qlik Management Console (QMC) + +Open your Qlik Management Console (QMC) and navigate to certificates section. + +{% image + src="/images/v1.1.1/connectors/qliksense/qlik-certificate-nav.png" + alt="Navigate to certificates in QMC" + caption="Navigate to certificates in QMC" + /%} + +# Step 2: Provide Details and Export Certificates + +1. In the Machine name box, type the full computer name of the computer that you are creating the certificates for: MYMACHINE.mydomain.com or the IP address. + +2. Using a password is optional. If you choose to use a password, the same password applies to exported client and server certificates. + a. Type a password in the Certificate password box. + b. Repeat the password in the Retype password box. + The passwords must match. + +3. Select Include secret key if you want to add a secret key to the public key. + +4. From the *Export file format for certificates* field select the "Platform independent PEM-format" + +{% image + src="/images/v1.1.1/connectors/qliksense/qlik-export-cert.png" + alt="Provide Certificate Details" + caption="Provide Certificate Details" + /%} + + +# Step 3: Locate the certificates + +Once you have exported the certificates you can see the location of exported certificates just below the certificate details page. When you navigate to that location you will find the `root.pem`, `client.pem` & `client_key.pem` certificates which will be used by OpenMetadata. + +{% image + src="/images/v1.1.1/connectors/qliksense/qlik-locate-certificates.png" + alt="Locate Certificate" + caption="Locate Certificate" + /%} \ No newline at end of file diff --git a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/qliksense/index.md b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/qliksense/index.md new file mode 100644 index 00000000000..b0adfc4663f --- /dev/null +++ b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/qliksense/index.md @@ -0,0 +1,98 @@ +--- +title: Qlik Sense +slug: /connectors/dashboard/qliksense +--- + +# Qlik Sense + +| Stage | PROD | +|------------|------------------------------| +| Dashboards | {% icon iconName="check" /%} | +| Charts | {% icon iconName="check" /%} | +| Owners | {% icon iconName="cross" /%} | +| Tags | {% icon iconName="cross" /%} | +| Datamodels | {% icon iconName="check" /%} | +| Lineage | {% icon iconName="check" /%} | + +In this section, we provide guides and references to use the Qlik Sense connector. + +Configure and schedule Metabase metadata and profiler workflows from the OpenMetadata UI: + +- [Requirements](#requirements) +- [Metadata Ingestion](#metadata-ingestion) + +{% partial file="/v1.1.1/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/dashboard/qliksense/yaml"} /%} + +## Requirements + +{%inlineCallout icon="description" bold="OpenMetadata 1.1.1 or later" href="/deployment"%} +To deploy OpenMetadata, check the Deployment guides. +{%/inlineCallout%} + +## Metadata Ingestion + +{% partial + file="/v1.1.1/connectors/metadata-ingestion-ui.md" + variables={ + connector: "QlikSense", + selectServicePath: "/images/v1.1.1/connectors/qliksense/select-service.png", + addNewServicePath: "/images/v1.1.1/connectors/qliksense/add-new-service.png", + serviceConnectionPath: "/images/v1.1.1/connectors/qliksense/service-connection.png", +} +/%} + +{% stepsContainer %} +{% extraContent parentTagName="stepsContainer" %} + +#### Connection Details + +- **Qlik Sense Base URL**: This field refers to the base url of your Qlik Sense Portal, will be used for generating the redirect links for dashboards and charts. Example: `https://server.domain.com` or `https://server.domain.com/` +- **Qlik Engine JSON API Websocket URL**: Enter the websocket url of Qlik Sense Engine JSON API. Refer to [this](https://help.qlik.com/en-US/sense-developer/May2023/Subsystems/EngineAPI/Content/Sense_EngineAPI/GettingStarted/connecting-to-engine-api.htm) document for more details about. Example: `wss://server.domain.com:4747` or `wss://server.domain.com[/virtual proxy]`. + +Since we use the Qlik Sense Engine APIs, we need to authenticate to those APIs using certificates generated on Qlik Management Console. + +**Qlik Certificate By Values**: In this approach we provide the content of the certificates to the relevant field. + - **Client Certificate Value**: This field specifies the value of `client.pem` certificate required for authentication. + - **Client Key Certificate Value**: This field specifies the value of `client_key.pem` certificate required for authentication. + - **Root Certificate Value**: This field specifies the value of `root.pem` certificate required for authentication. + - **Staging Directory Path**: This field specifies the path to temporary staging directory, where the certificates will be stored temporarily during the ingestion process, which will de deleted once the ingestion job is over. + +when you are using this approach make sure you are passing the key in a correct format. If your certificate looks like this: + +``` +-----BEGIN CERTIFICATE----- +MII.. +MBQ... +CgU.. +8Lt.. +... +h+4= +-----END CERTIFICATE----- +``` + +You will have to replace new lines with `\n` and the final private key that you need to pass should look like this: + +``` +-----BEGIN CERTIFICATE-----\nMII..\nMBQ...\nCgU..\n8Lt..\n...\nh+4=\n-----END CERTIFICATE-----\n +``` + +**Qlik Certificate By Path**: In this approach we provide the path of the certificates to the certificate stored in the container or environment running the ingestion workflow. + - **Client Certificate Path**: This field specifies the path of `client.pem` certificate required for authentication. + - **Client Key Certificate Value**: This field specifies the path of `client_key.pem` certificate required for authentication. + - **Root Certificate Value**: This field specifies the path of `root.pem` certificate required for authentication. + +**User Directory**: This field specifies the user directory of the user. + +**User ID**: This field specifies the user id of the user. + +{% /extraContent %} + +{% partial file="/v1.1.1/connectors/test-connection.md" /%} + +{% partial file="/v1.1.1/connectors/dashboard/configure-ingestion.md" /%} + +{% partial file="/v1.1.1/connectors/ingestion-schedule-and-deploy.md" /%} + +{% /stepsContainer %} + +{% partial file="/v1.1.1/connectors/troubleshooting.md" /%} diff --git a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/qliksense/yaml.md b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/qliksense/yaml.md new file mode 100644 index 00000000000..c7361d652bf --- /dev/null +++ b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/dashboard/qliksense/yaml.md @@ -0,0 +1,246 @@ +--- +title: Run the Qlik Sense Connector Externally +slug: /connectors/dashboard/qliksense/yaml +--- + +# Run the PowerBI Connector Externally + +| Stage | PROD | +|------------|------------------------------| +| Dashboards | {% icon iconName="check" /%} | +| Charts | {% icon iconName="check" /%} | +| Owners | {% icon iconName="cross" /%} | +| Tags | {% icon iconName="cross" /%} | +| Datamodels | {% icon iconName="check" /%} | +| Lineage | {% icon iconName="check" /%} | + +In this section, we provide guides and references to use the PowerBI connector. + +Configure and schedule PowerBI metadata and profiler workflows from the OpenMetadata UI: + +- [Requirements](#requirements) +- [Metadata Ingestion](#metadata-ingestion) + +{% partial file="/v1.1.1/connectors/external-ingestion-deployment.md" /%} + +## Requirements + +{%inlineCallout icon="description" bold="OpenMetadata 0.12 or later" href="/deployment"%} +To deploy OpenMetadata, check the Deployment guides. +{%/inlineCallout%} + +### Python Requirements + +To run the PowerBI ingestion, you will need to install: + +```bash +pip3 install "openmetadata-ingestion[qliksense]" +``` + +## Metadata Ingestion + +All connectors are defined as JSON Schemas. +[Here](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/dashboard/qlikSenseConnection.json) +you can find the structure to create a connection to QlikSense. + +In order to create and run a Metadata Ingestion workflow, we will follow +the steps to create a YAML configuration able to connect to the source, +process the Entities if needed, and reach the OpenMetadata server. + +The workflow is modeled around the following +[JSON Schema](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json) + +### 1. Define the YAML Config + +This is a sample config for Qlik Sense: + +{% codePreview %} + +{% codeInfoContainer %} + +#### Source Configuration - Service Connection + +{% codeInfo srNumber=1 %} + +**hostPort**: Qlik Engine JSON API Websocket URL + +Enter the websocket url of Qlik Sense Engine JSON API. Refer to [this](https://help.qlik.com/en-US/sense-developer/May2023/Subsystems/EngineAPI/Content/Sense_EngineAPI/GettingStarted/connecting-to-engine-api.htm) document for more details about + +Example: `wss://server.domain.com:4747` or `wss://server.domain.com[/virtual proxy]` + +**Note:** Notice that you have to provide the websocket url here which would begin with either `wss://` or `ws://` + +{% /codeInfo %} + +{% codeInfo srNumber=2 %} + +**displayUrl**: Qlik Sense Base URL + +This field refers to the base url of your Qlik Sense Portal, will be used for generating the redirect links for dashboards and charts. + +Example: `https://server.domain.com` or `https://server.domain.com/` + +{% /codeInfo %} + +{% codeInfo srNumber=3 %} + +Since we use the Qlik Sense Engine APIs, we need to authenticate to those APIs using certificates generated on Qlik Management Console. + +In this approach we provide the path of the certificates to the certificate stored in the container or environment running the ingestion workflow. + +- **clientCertificate**: This field specifies the path of `client.pem` certificate required for authentication. +- **clientKeyCertificate**: This field specifies the path of `client_key.pem` certificate required for authentication. +- **rootCertificate**: This field specifies the path of `root.pem` certificate required for authentication. + +{% /codeInfo %} + +{% codeInfo srNumber=4 %} + +In this approach we provide the content of the certificates to the relevant field. + +- **Client Certificate Value**: This field specifies the value of `client.pem` certificate required for authentication. +- **Client Key Certificate Value**: This field specifies the value of `client_key.pem` certificate required for authentication. +- **Root Certificate Value**: This field specifies the value of `root.pem` certificate required for authentication. +- **Staging Directory Path**: This field specifies the path to temporary staging directory, where the certificates will be stored temporarily during the ingestion process, which will de deleted once the ingestion job is over. + +when you are using this approach make sure you are passing the key in a correct format. If your certificate looks like this: + +``` +-----BEGIN CERTIFICATE----- +MII.. +MBQ... +CgU.. +8Lt.. +... +h+4= +-----END CERTIFICATE----- +``` + +You will have to replace new lines with `\n` and the final private key that you need to pass should look like this: + +``` +-----BEGIN CERTIFICATE-----\nMII..\nMBQ...\nCgU..\n8Lt..\n...\nh+4=\n-----END CERTIFICATE-----\n +``` + + +{% /codeInfo %} + +{% codeInfo srNumber=5 %} + +**userId**: This field specifies the user directory of the user. + +{% /codeInfo %} + +{% codeInfo srNumber=6 %} + +**userDirectory**: This field specifies the user directory of the user. + +{% /codeInfo %} + +#### Source Configuration - Source Config + +{% codeInfo srNumber=9 %} + +The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json): + +- **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it. +- **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard". +- **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten. +- **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion. +- **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion. +- **markDeletedDashboards**: Set the 'Mark Deleted Dashboards' toggle to flag dashboards as soft-deleted if they are not present anymore in the source system. + +{% /codeInfo %} + +#### Sink Configuration + +{% codeInfo srNumber=10 %} + +To send the metadata to OpenMetadata, it needs to be specified as `type: metadata-rest`. + +{% /codeInfo %} + +{% partial file="/v1.1.1/connectors/workflow-config.md" /%} + +{% /codeInfoContainer %} + +{% codeBlock fileName="filename.yaml" %} + +```yaml +source: + type: qliksense + serviceName: local_qliksense + serviceConnection: + config: + type: QlikSense +``` +```yaml {% srNumber=1 %} + hostPort: wss://localhost:4747 +``` +```yaml {% srNumber=2 %} + displayUrl: https://localhost +``` +```yaml {% srNumber=3 %} + certificates: + # pass certificate paths + clientCertificate: /path/to/client.pem + clientKeyCertificate: /path/to/client_key.pem + rootCertificate: /path/to/root.pem +``` +```yaml {% srNumber=4 %} + # pass certificate values + # clientCertificateData: -----BEGIN CERTIFICATE-----\n....\n.....\n-----END CERTIFICATE-----\n + # clientKeyCertificateData: -----BEGIN RSA PRIVATE KEY-----\n....\n....\n-----END RSA PRIVATE KEY-----\n + # rootCertificateData: -----BEGIN CERTIFICATE-----\n....\n...-----END CERTIFICATE-----\n + # stagingDir: /tmp/stage +``` +```yaml {% srNumber=5 %} + userId: user_id +``` +```yaml {% srNumber=6 %} + userDirectory: user_dir +``` +```yaml {% srNumber=7 %} + sourceConfig: + config: + type: DashboardMetadata + # dbServiceNames: + # - service1 + # - service2 + # dashboardFilterPattern: + # includes: + # - dashboard1 + # - dashboard2 + # excludes: + # - dashboard3 + # - dashboard4 + # chartFilterPattern: + # includes: + # - chart1 + # - chart2 + # excludes: + # - chart3 + # - chart4 +``` +```yaml {% srNumber=8 %} +sink: + type: metadata-rest + config: {} +``` + +{% partial file="workflow-config-yaml.md" /%} + +{% /codeBlock %} + +{% /codePreview %} + +### 2. Run with the CLI + +First, we will need to save the YAML file. Afterward, and with all requirements installed, we can run: + +```bash +metadata ingest -c +``` + +Note that from connector to connector, this recipe will always be the same. By updating the YAML configuration, +you will be able to extract metadata from different sources. diff --git a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/index.md b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/index.md index 5bca348bcb6..ac1493be3a8 100644 --- a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/index.md +++ b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/index.md @@ -69,6 +69,8 @@ the following docs to run the Ingestion Framework in any orchestrator externally - [Superset](/connectors/dashboard/superset) - [Tableau](/connectors/dashboard/tableau) - [Domo Dashboard](/connectors/dashboard/domo-dashboard) +- [Qlik Sense](/connectors/dashboard/qliksense) +- [QuickSight](/connectors/dashboard/quicksight) ## Messaging Services diff --git a/openmetadata-docs/content/v1.1.1-SNAPSHOT/menu.md b/openmetadata-docs/content/v1.1.1-SNAPSHOT/menu.md index 57296f1f500..a37d792e7a1 100644 --- a/openmetadata-docs/content/v1.1.1-SNAPSHOT/menu.md +++ b/openmetadata-docs/content/v1.1.1-SNAPSHOT/menu.md @@ -378,6 +378,12 @@ site_menu: url: /connectors/dashboard/quicksight - category: Connectors / Dashboard / QuickSight / Run Externally url: /connectors/dashboard/quicksight/yaml + - category: Connectors / Dashboard / Qlik Sense + url: /connectors/dashboard/qliksense + - category: Connectors / Dashboard / Qlik Sense / Run Externally + url: /connectors/dashboard/qliksense/yaml + - category: Connectors / Dashboard / Qlik Sense / Export Certificates + url: /connectors/dashboard/qliksense/certificates - category: Connectors / Dashboard / Redash url: /connectors/dashboard/redash - category: Connectors / Dashboard / Redash / Run Externally diff --git a/openmetadata-docs/images/v1.1.1/connectors/qliksense/add-new-service.png b/openmetadata-docs/images/v1.1.1/connectors/qliksense/add-new-service.png new file mode 100644 index 00000000000..92e93a1c89f Binary files /dev/null and b/openmetadata-docs/images/v1.1.1/connectors/qliksense/add-new-service.png differ diff --git a/openmetadata-docs/images/v1.1.1/connectors/qliksense/qlik-certificate-nav.png b/openmetadata-docs/images/v1.1.1/connectors/qliksense/qlik-certificate-nav.png new file mode 100644 index 00000000000..8e5accd2bd0 Binary files /dev/null and b/openmetadata-docs/images/v1.1.1/connectors/qliksense/qlik-certificate-nav.png differ diff --git a/openmetadata-docs/images/v1.1.1/connectors/qliksense/qlik-export-cert.png b/openmetadata-docs/images/v1.1.1/connectors/qliksense/qlik-export-cert.png new file mode 100644 index 00000000000..1ce221c1951 Binary files /dev/null and b/openmetadata-docs/images/v1.1.1/connectors/qliksense/qlik-export-cert.png differ diff --git a/openmetadata-docs/images/v1.1.1/connectors/qliksense/qlik-locate-certificates.png b/openmetadata-docs/images/v1.1.1/connectors/qliksense/qlik-locate-certificates.png new file mode 100644 index 00000000000..0596ca54140 Binary files /dev/null and b/openmetadata-docs/images/v1.1.1/connectors/qliksense/qlik-locate-certificates.png differ diff --git a/openmetadata-docs/images/v1.1.1/connectors/qliksense/select-service.png b/openmetadata-docs/images/v1.1.1/connectors/qliksense/select-service.png new file mode 100644 index 00000000000..62e834692bb Binary files /dev/null and b/openmetadata-docs/images/v1.1.1/connectors/qliksense/select-service.png differ diff --git a/openmetadata-docs/images/v1.1.1/connectors/qliksense/service-connection.png b/openmetadata-docs/images/v1.1.1/connectors/qliksense/service-connection.png new file mode 100644 index 00000000000..c8d2095adef Binary files /dev/null and b/openmetadata-docs/images/v1.1.1/connectors/qliksense/service-connection.png differ diff --git a/openmetadata-service/src/main/resources/json/data/testConnections/dashboard/qliksense.json b/openmetadata-service/src/main/resources/json/data/testConnections/dashboard/qliksense.json new file mode 100644 index 00000000000..5c4007f5d3a --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/testConnections/dashboard/qliksense.json @@ -0,0 +1,14 @@ +{ + "name": "QlikSense", + "displayName": "QlikSense Test Connection", + "description": "This Test Connection validates the access against the server and basic metadata extraction of dashboards and charts.", + "steps": [ + { + "name": "GetDashboards", + "description": "List all the dashboards available to the user", + "errorMessage": "Failed to fetch dashboards, please validate the credentials or validate if user has access to fetch dashboards", + "shortCircuit": true, + "mandatory": true + } + ] + } \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json index 9b818c1397a..1210a77d986 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json @@ -19,7 +19,8 @@ "MetabaseDataModel", "LookMlView", "LookMlExplore", - "PowerBIDataModel" + "PowerBIDataModel", + "QlikSenseDataModel" ], "javaEnums": [ { @@ -39,6 +40,9 @@ }, { "name": "PowerBIDataModel" + }, + { + "name": "QlikSenseDataModel" } ] } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/dashboard/qlikSenseConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/dashboard/qlikSenseConnection.json new file mode 100644 index 00000000000..e58788ceaca --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/dashboard/qlikSenseConnection.json @@ -0,0 +1,133 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/dashboard/qlikSenseConnection.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "QlikSenseConnection", + "description": "Qlik Sense Connection Config", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.dashboard.QlikSenseConnection", + "definitions": { + "qlikSenseType": { + "description": "Qlik sense service type", + "type": "string", + "enum": [ + "QlikSense" + ], + "default": "QlikSense" + }, + "qlikCertificatePath": { + "description": "Qlik Authentication Certificate File Path", + "title": "Qlik Certificates By File Path", + "type": "object", + "properties": { + "clientCertificate": { + "title": "Client Certificate Path", + "description": "Client Certificate", + "type": "string" + }, + "clientKeyCertificate": { + "title": "Client Key Certificate", + "description": "Client Key Certificate.", + "type": "string" + }, + "rootCertificate": { + "title": "Root Certificate", + "description": "Root Certificate.", + "type": "string" + } + }, + "required": [ + "clientCertificate", + "clientKeyCertificate", + "rootCertificate" + ] + }, + "qlikCertificateValues": { + "description": "Qlik Authentication Certificate By Values", + "title": "Qlik Certificates By Values", + "type": "object", + "properties": { + "clientCertificateData": { + "title": "Client Certificate Value", + "description": "Client Certificate", + "type": "string", + "format": "password" + }, + "clientKeyCertificateData": { + "title": "Client Key Certificate Value", + "description": "Client Key Certificate.", + "type": "string", + "format": "password" + }, + "rootCertificateData": { + "title": "Root Certificate Value", + "description": "Root Certificate.", + "type": "string", + "format": "password" + }, + "stagingDir": { + "title": "Staging Directory Path", + "description": "Staging Directory Path", + "type": "string", + "default": "/tmp/openmetadata-qlik" + } + }, + "required": [ + "clientCertificateData", + "clientKeyCertificateData", + "rootCertificateData", + "stagingDir" + ] + } + }, + "properties": { + "type": { + "title": "Service Type", + "description": "Service Type", + "$ref": "#/definitions/qlikSenseType", + "default": "QlikSense" + }, + "displayUrl": { + "expose": true, + "title": "Qlik Sense Base URL", + "description": "Qlik Sense Base URL, used for genrating dashboard & chat url", + "type": "string", + "format": "uri" + }, + "hostPort": { + "expose": true, + "title": "Qlik Engine JSON API Websocket URL", + "description": "URL for the superset instance.", + "type": "string", + "format": "uri" + }, + "certificates": { + "oneOf": [ + { + "$ref": "#/definitions/qlikCertificateValues" + }, + { + "$ref": "#/definitions/qlikCertificatePath" + } + ] + }, + "userDirectory": { + "title": "User Directory", + "description": "User Directory.", + "type": "string" + }, + "userId": { + "title": "User ID", + "description": "User ID.", + "type": "string" + }, + "supportsMetadataExtraction": { + "title": "Supports Metadata Extraction", + "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" + } + }, + "additionalProperties": false, + "required": [ + "hostPort", + "certificates" + ] +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json index 6b236f77a36..dd6d85c41c8 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json @@ -24,7 +24,8 @@ "Mode", "CustomDashboard", "DomoDashboard", - "QuickSight" + "QuickSight", + "QlikSense" ], "javaEnums": [ { @@ -56,6 +57,9 @@ }, { "name": "QuickSight" + }, + { + "name": "QlikSense" } ] }, @@ -99,6 +103,9 @@ }, { "$ref": "./connections/dashboard/quickSightConnection.json" + }, + { + "$ref": "./connections/dashboard/qlikSenseConnection.json" } ] } diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/QlikSense.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/QlikSense.md new file mode 100644 index 00000000000..51d6944086f --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/QlikSense.md @@ -0,0 +1,156 @@ +# QlikSense + +In this section, we provide guides and references to use the Metabase connector. + +## Requirements + +We will extract the metadata using the [Qlik Sense Engine JSON API](https://help.qlik.com/en-US/sense-developer/May2023/Subsystems/EngineAPI/Content/Sense_EngineAPI/introducing-engine-API.htm). + +You can find further information on the Qlik Sense connector in the [docs](https://docs.open-metadata.org/connectors/dashboard/qliksense). + +## Connection Details + +$$section +### Qlik Sense Base URL $(id="displayUrl") + +This field refers to the base url of your Qlik Sense Portal, will be used for generating the redirect links for dashboards and charts. + +Example: `https://server.domain.com` or `https://server.domain.com/` +$$ + +$$section +### Qlik Engine JSON API Websocket URL $(id="hostPort") + +Enter the websocket url of Qlik Sense Engine JSON API. Refer to [this](https://help.qlik.com/en-US/sense-developer/May2023/Subsystems/EngineAPI/Content/Sense_EngineAPI/GettingStarted/connecting-to-engine-api.htm) document for more details about + +Example: `wss://server.domain.com:4747` or `wss://server.domain.com[/virtual proxy]` + +**Note:** Notice that you have to provide the websocket url here which would begin with either `wss://` or `ws://` +$$ + +$$section +### Client Certificate Value $(id="clientCertificateData") + +This field specifies the value of `client.pem` certificate required for authentication. + + +Make sure you are passing the key in a correct format. If your certificate looks like this: + +``` +-----BEGIN CERTIFICATE----- +MII.. +MBQ... +CgU.. +8Lt.. +... +h+4= +-----END CERTIFICATE----- +``` + +You will have to replace new lines with `\n` and the final private key that you need to pass should look like this: + +``` +-----BEGIN CERTIFICATE-----\nMII..\nMBQ...\nCgU..\n8Lt..\n...\nh+4=\n-----END CERTIFICATE-----\n +``` +$$ + + +$$section +### Client Key Certificate Value $(id="clientKeyCertificateData") + +This field specifies the value of `client_key.pem` certificate required for authentication. + + +Make sure you are passing the key in a correct format. If your certificate looks like this: + +``` +-----BEGIN RSA PRIVATE KEY----- +MII.. +MBQ... +CgU.. +8Lt.. +... +h+4= +-----END RSA PRIVATE KEY----- +``` + +You will have to replace new lines with `\n` and the final private key that you need to pass should look like this: + +``` +-----BEGIN CERTIFICATE-----\nMII..\nMBQ...\nCgU..\n8Lt..\n...\nh+4=\n-----END CERTIFICATE-----\n +``` +$$ + + +$$section +### Root Certificate Value $(id="rootCertificateData") + +This field specifies the value of `root.pem` certificate required for authentication. + + +Make sure you are passing the key in a correct format. If your certificate looks like this: + +``` +-----BEGIN CERTIFICATE----- +MII.. +MBQ... +CgU.. +8Lt.. +... +h+4= +-----END CERTIFICATE----- +``` + +You will have to replace new lines with `\n` and the final private key that you need to pass should look like this: + +``` +-----BEGIN CERTIFICATE-----\nMII..\nMBQ...\nCgU..\n8Lt..\n...\nh+4=\n-----END CERTIFICATE-----\n +``` +$$ + + +$$section +### Staging Directory Path $(id="stagingDir") + +This field specifies the path to temporary staging directory, where the certificates will be stored temporarily during the ingestion process, which will de deleted once the ingestion job is over. +$$ + +$$section +### Client Certificate Path $(id="clientCertificate") + +This field specifies the path of `client.pem` certificate required for authentication. + +Example: `/path/to/client.pem` +$$ + + +$$section +### Client Key Certificate Path $(id="clientKeyCertificate") + +This field specifies the path of `client_key.pem` certificate required for authentication. + +Example: `/path/to/client_key.pem` +$$ + + +$$section +### Root Certificate Path $(id="rootCertificate") + +This field specifies the path of `root.pem` certificate required for authentication. + +Example: `/path/to/root.pem` +$$ + +$$section +### User Directory $(id="userDirectory") + +This field specifies the user directory of the user. +$$ + + + +$$section +### User ID $(id="userId") + +This field specifies the user id of the user. +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/img/service-icon-qlik-sense.png b/openmetadata-ui/src/main/resources/ui/src/assets/img/service-icon-qlik-sense.png new file mode 100644 index 00000000000..6327edc3466 Binary files /dev/null and b/openmetadata-ui/src/main/resources/ui/src/assets/img/service-icon-qlik-sense.png differ diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts index 8ac075e9e30..1ea1756e382 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts @@ -59,6 +59,7 @@ import powerbi from '../assets/img/service-icon-power-bi.png'; import prefect from '../assets/img/service-icon-prefect.png'; import presto from '../assets/img/service-icon-presto.png'; import pulsar from '../assets/img/service-icon-pulsar.png'; +import qlikSense from '../assets/img/service-icon-qlik-sense.png'; import query from '../assets/img/service-icon-query.png'; import quicksight from '../assets/img/service-icon-quicksight.png'; import redash from '../assets/img/service-icon-redash.png'; @@ -171,6 +172,7 @@ export const GCS = gcs; export const MS_AZURE = msAzure; export const SPLINE = spline; export const MONGODB = mongodb; +export const QLIK_SENSE = qlikSense; export const PLUS = plus; export const NOSERVICE = noService; @@ -350,6 +352,7 @@ export const BETA_SERVICES = [ DatabaseServiceType.SapHana, PipelineServiceType.Spline, DatabaseServiceType.MongoDB, + DashboardServiceType.QlikSense, ]; export const TEST_CONNECTION_INITIAL_MESSAGE = i18n.t( diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/DashboardServiceUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/DashboardServiceUtils.ts index f927798f51f..683bd311ce8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/DashboardServiceUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/DashboardServiceUtils.ts @@ -23,6 +23,7 @@ import lookerConnection from '../jsons/connectionSchemas/connections/dashboard/l import metabaseConnection from '../jsons/connectionSchemas/connections/dashboard/metabaseConnection.json'; import modeConnection from '../jsons/connectionSchemas/connections/dashboard/modeConnection.json'; import powerBIConnection from '../jsons/connectionSchemas/connections/dashboard/powerBIConnection.json'; +import qliksenseConnection from '../jsons/connectionSchemas/connections/dashboard/qlikSenseConnection.json'; import quicksightConnection from '../jsons/connectionSchemas/connections/dashboard/quickSightConnection.json'; import redashConnection from '../jsons/connectionSchemas/connections/dashboard/redashConnection.json'; import tableauConnection from '../jsons/connectionSchemas/connections/dashboard/tableauConnection.json'; @@ -87,6 +88,11 @@ export const getDashboardConfig = (type: DashboardServiceType) => { case DashboardServiceType.QuickSight: { schema = quicksightConnection; + break; + } + case DashboardServiceType.QlikSense: { + schema = qliksenseConnection; + break; } } diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx index cd917543435..b665acdc8ea 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx @@ -67,6 +67,7 @@ import { POSTGRES, POWERBI, PRESTO, + QLIK_SENSE, QUICKSIGHT, REDASH, REDPANDA, @@ -245,6 +246,9 @@ export const serviceTypeLogo = (type: string) => { case DashboardServiceType.Mode: return MODE; + case DashboardServiceType.QlikSense: + return QLIK_SENSE; + case PipelineServiceType.Airflow: return AIRFLOW;