diff --git a/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/client.py b/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/client.py index 15a277852c0..0d461583256 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/client.py @@ -266,3 +266,24 @@ class MicroStrategyClient: logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}") return None + + def get_cube_sql_details(self, project_id: str, cube_id: str) -> Optional[str]: + """ + Get Cube SQL Details + """ + try: + headers = { + "X-MSTR-ProjectID": project_id, + "cubeId": cube_id, + } | self.auth_params.auth_header + + resp_dataset = self.client._request( # pylint: disable=protected-access + "GET", path=f"/v2/cubes/{cube_id}/sqlView", headers=headers + ) + return resp_dataset["sqlStatement"] + + except Exception: + logger.debug(traceback.format_exc()) + logger.warning(f"Failed to fetch the cube with id: {cube_id}") + + return None diff --git a/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/helpers.py b/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/helpers.py new file mode 100644 index 00000000000..27bf39e59c6 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/helpers.py @@ -0,0 +1,67 @@ +# Copyright 2025 Collate +# Licensed under the Collate Community License, Version 1.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Microstrategy source helpers. +""" +from __future__ import annotations + +from typing import Any, Dict + +from metadata.generated.schema.entity.data.table import Column, DataType + + +class MicroStrategyColumnParser: + """ + Responsible for containing the logic to parse a column from MicroStrategy to OpenMetadata + """ + + datatype_mapping = { + "big decimal": DataType.DECIMAL, + "binary": DataType.BYTES, + "char": DataType.CHAR, + "date": DataType.DATE, + "decimal": DataType.DECIMAL, + "double": DataType.DOUBLE, + "float": DataType.FLOAT, + "integer": DataType.INT, + "longvarbin": DataType.BLOB, + "longvarchar": DataType.CLOB, + "nchar": DataType.CHAR, + "numeric": DataType.NUMERIC, + "nvarchar": DataType.VARCHAR, + "real": DataType.STRING, + "time": DataType.TIME, + "timestamp": DataType.TIMESTAMP, + "unsigned": DataType.INT, + "varbin": DataType.BYTES, + "varchar": DataType.VARCHAR, + "utf8char": DataType.CHAR, + } + + @classmethod + def parse(cls, field: Dict[str, Any]) -> Column: + """ + Parses a MicroStrategy table column into an OpenMetadata column. + """ + + array_data_type = None + data_type = cls.datatype_mapping.get( + field["dataType"].lower(), DataType.UNKNOWN + ) + + column_def = { + "name": field["name"], + "dataTypeDisplay": field["dataType"], + "dataType": data_type, + "arrayDataType": array_data_type, + } + + return Column(**column_def) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/metadata.py index f885e1078d7..70c99b5aa9d 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/metadata.py @@ -14,11 +14,23 @@ from typing import Iterable, List, Optional from metadata.generated.schema.api.data.createChart import CreateChartRequest from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest +from metadata.generated.schema.api.data.createDashboardDataModel import ( + CreateDashboardDataModelRequest, +) from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.entity.data.chart import Chart +from metadata.generated.schema.entity.data.dashboardDataModel import ( + DashboardDataModel, + DataModelType, +) +from metadata.generated.schema.entity.data.table import Column, DataType from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import ( MicroStrategyConnection, ) +from metadata.generated.schema.entity.services.dashboardService import ( + DashboardServiceType, +) +from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -29,18 +41,29 @@ from metadata.generated.schema.type.basic import ( EntityName, FullyQualifiedEntityName, SourceUrl, + Uuid, ) +from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails +from metadata.generated.schema.type.entityLineage import Source as LineageSource +from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.api.models import Either from metadata.ingestion.api.steps import InvalidSourceException +from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper +from metadata.ingestion.lineage.parser import LineageParser +from metadata.ingestion.lineage.sql_lineage import get_table_entities_from_query from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource +from metadata.ingestion.source.dashboard.microstrategy.helpers import ( + MicroStrategyColumnParser, +) from metadata.ingestion.source.dashboard.microstrategy.models import ( MstrDashboard, MstrDashboardDetails, + MstrDataset, MstrPage, ) from metadata.utils import fqn -from metadata.utils.filters import filter_by_chart +from metadata.utils.filters import filter_by_chart, filter_by_datamodel from metadata.utils.helpers import clean_uri, get_standard_chart_type from metadata.utils.logger import ingestion_logger @@ -163,7 +186,75 @@ class MicrostrategySource(DashboardServiceSource): dashboard_details: MstrDashboardDetails, db_service_name: Optional[str] = None, ) -> Optional[Iterable[AddLineageRequest]]: - """Not Implemented""" + """ + Get lineage between datamodel and data sources + """ + if not db_service_name: + return + + database_service = self.metadata.get_by_name( + entity=DatabaseService, fqn=db_service_name + ) + dialect = ConnectionTypeDialectMapper.dialect_of( + database_service.serviceType.value + ) + + for dataset in dashboard_details.datasets: + cube_sql = self.client.get_cube_sql_details( + dashboard_details.projectId, dataset.id + ) + if not cube_sql: + continue + + datamodel_fqn = fqn.build( + self.metadata, + entity_type=DashboardDataModel, + service_name=self.context.get().dashboard_service, + data_model_name=dataset.id, + ) + datamodel_entity = self.metadata.get_by_name( + entity=DashboardDataModel, fqn=datamodel_fqn + ) + + try: + lineage_parser = LineageParser(cube_sql, dialect=dialect) + for table in lineage_parser.source_tables: + table_entities = get_table_entities_from_query( + metadata=self.metadata, + service_name=db_service_name, + database_name="*", + database_schema="*", + table_name=str(table), + ) + if not table_entities: + logger.debug(f"Table not found in metadata: {str(table)}") + continue + for table_entity in table_entities or []: + yield Either( + right=AddLineageRequest( + edge=EntitiesEdge( + fromEntity=EntityReference( + id=Uuid(table_entity.id.root), + type="table", + ), + toEntity=EntityReference( + id=Uuid(datamodel_entity.id.root), + type="dashboardDataModel", + ), + lineageDetails=LineageDetails( + source=LineageSource.DashboardLineage + ), + ) + ) + ) + except Exception as exc: + yield Either( + left=StackTraceError( + name="Dashboard Lineage", + error=f"Error to yield dashboard lineage details: {exc}", + stackTrace=traceback.format_exc(), + ) + ) def yield_dashboard_chart( self, dashboard_details: MstrDashboardDetails @@ -213,6 +304,74 @@ class MicrostrategySource(DashboardServiceSource): ) ) + def _get_column_info(self, dataset: MstrDataset) -> Optional[List[Column]]: + """Build columns from dataset""" + datasource_columns = [] + for available_object in dataset.availableObjects or []: + try: + parsed_column = { + "dataTypeDisplay": available_object.type.title(), + "dataType": DataType.UNKNOWN, + "name": available_object.name, + "displayName": available_object.name, + } + parsed_column_children = [] + for form in available_object.forms or []: + parsed_column_children.append(MicroStrategyColumnParser.parse(form)) + if parsed_column_children: + parsed_column["children"] = parsed_column_children + + datasource_columns.append(Column(**parsed_column)) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Error to yield datamodel column: {exc}") + return datasource_columns + + def yield_datamodel( + self, dashboard_details: MstrDashboardDetails + ) -> Optional[Iterable[CreateDashboardDataModelRequest]]: + """Get datamodel method + + Args: + dashboard_details: + Returns: + Iterable[CreateDashboardDataModelRequest] + """ + try: + if self.source_config.includeDataModels: + for dataset in dashboard_details.datasets: + if filter_by_datamodel( + self.source_config.dataModelFilterPattern, dataset.name + ): + self.status.filter(dataset.name, "Data model filtered out.") + continue + data_model_type = DataModelType.MicroStrategyDataset.value + datamodel_columns = self._get_column_info(dataset) + + data_model_request = CreateDashboardDataModelRequest( + name=EntityName(dataset.id), + displayName=dataset.name, + service=FullyQualifiedEntityName( + self.context.get().dashboard_service + ), + dataModelType=data_model_type, + serviceType=DashboardServiceType.MicroStrategy.value, + columns=datamodel_columns, + project=self.get_project_name( + dashboard_details=dashboard_details + ), + ) + yield Either(right=data_model_request) + self.register_record_datamodel(datamodel_request=data_model_request) + except Exception as exc: + yield Either( + left=StackTraceError( + name=dataset.name, + error=f"Error yielding Data Model [{dataset.name}]: {exc}", + stackTrace=traceback.format_exc(), + ) + ) + def close(self): # close the api session self.client.close_api_session() diff --git a/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/models.py b/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/models.py index 26686f2a3ba..41a6cfb40e6 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/models.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/models.py @@ -12,7 +12,7 @@ MicroStrategy Models """ from datetime import datetime -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional from pydantic import BaseModel @@ -134,6 +134,17 @@ class MstrAvailableObject(BaseModel): id: str name: str type: str + forms: Optional[List[Dict[str, Any]]] = None + + +class MstrDataset(BaseModel): + id: str + name: str + availableObjects: Optional[List[MstrAvailableObject]] = None + rows: Optional[List[Dict[str, Any]]] = None + columns: Optional[List[Dict[str, Any]]] = None + pageBy: Optional[List[Dict[str, Any]]] = None + sqlStatement: Optional[str] = None class MstrDashboardDetails(BaseModel): @@ -143,6 +154,7 @@ class MstrDashboardDetails(BaseModel): projectName: str currentChapter: str chapters: List[MstrChapter] + datasets: List[MstrDataset] class AuthHeaderCookie(BaseModel): diff --git a/openmetadata-docs/content/v1.7.x/connectors/dashboard/microstrategy/index.md b/openmetadata-docs/content/v1.7.x/connectors/dashboard/microstrategy/index.md index d7076b4e51b..74b0e2b60fd 100644 --- a/openmetadata-docs/content/v1.7.x/connectors/dashboard/microstrategy/index.md +++ b/openmetadata-docs/content/v1.7.x/connectors/dashboard/microstrategy/index.md @@ -7,8 +7,8 @@ slug: /connectors/dashboard/microstrategy name="MicroStrategy" stage="PROD" platform="OpenMetadata" - availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels"] - unavailableFeatures=["Tags", "Projects", "Lineage"] + availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels", "Lineage"] + unavailableFeatures=["Tags", "Projects"] / %} In this section, we provide guides and references to use the MicroStrategy connector. diff --git a/openmetadata-docs/content/v1.7.x/connectors/dashboard/microstrategy/yaml.md b/openmetadata-docs/content/v1.7.x/connectors/dashboard/microstrategy/yaml.md index 1b361086170..13c6545f8bd 100644 --- a/openmetadata-docs/content/v1.7.x/connectors/dashboard/microstrategy/yaml.md +++ b/openmetadata-docs/content/v1.7.x/connectors/dashboard/microstrategy/yaml.md @@ -7,8 +7,8 @@ slug: /connectors/dashboard/microstrategy/yaml name="MicroStrategy" stage="PROD" platform="OpenMetadata" - availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels"] - unavailableFeatures=["Tags", "Projects", "Lineage"] + availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels", "Lineage"] + unavailableFeatures=["Tags", "Projects"] / %} In this section, we provide guides and references to use the MicroStrategy connector. diff --git a/openmetadata-docs/content/v1.8.x-SNAPSHOT/connectors/dashboard/microstrategy/index.md b/openmetadata-docs/content/v1.8.x-SNAPSHOT/connectors/dashboard/microstrategy/index.md index 0530d0b9450..473f5082815 100644 --- a/openmetadata-docs/content/v1.8.x-SNAPSHOT/connectors/dashboard/microstrategy/index.md +++ b/openmetadata-docs/content/v1.8.x-SNAPSHOT/connectors/dashboard/microstrategy/index.md @@ -7,8 +7,8 @@ slug: /connectors/dashboard/microstrategy name="MicroStrategy" stage="PROD" platform="OpenMetadata" - availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels"] - unavailableFeatures=["Tags", "Projects", "Lineage"] + availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels", "Lineage"] + unavailableFeatures=["Tags", "Projects"] / %} In this section, we provide guides and references to use the MicroStrategy connector. diff --git a/openmetadata-docs/content/v1.8.x-SNAPSHOT/connectors/dashboard/microstrategy/yaml.md b/openmetadata-docs/content/v1.8.x-SNAPSHOT/connectors/dashboard/microstrategy/yaml.md index 32268718899..0e07412431b 100644 --- a/openmetadata-docs/content/v1.8.x-SNAPSHOT/connectors/dashboard/microstrategy/yaml.md +++ b/openmetadata-docs/content/v1.8.x-SNAPSHOT/connectors/dashboard/microstrategy/yaml.md @@ -7,8 +7,8 @@ slug: /connectors/dashboard/microstrategy/yaml name="MicroStrategy" stage="PROD" platform="OpenMetadata" - availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels"] - unavailableFeatures=["Tags", "Projects", "Lineage"] + availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels", "Lineage"] + unavailableFeatures=["Tags", "Projects"] / %} In this section, we provide guides and references to use the MicroStrategy connector. diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json index 78aaa937784..04c8bb8a36e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json @@ -6,7 +6,10 @@ "description": "Dashboard Data Model entity definition. Data models are the schemas used to build dashboards, charts, or other data assets.", "type": "object", "javaType": "org.openmetadata.schema.entity.data.DashboardDataModel", - "javaInterfaces": ["org.openmetadata.schema.EntityInterface", "org.openmetadata.schema.ColumnsEntityInterface"], + "javaInterfaces": [ + "org.openmetadata.schema.EntityInterface", + "org.openmetadata.schema.ColumnsEntityInterface" + ], "definitions": { "dataModelType": { "javaType": "org.openmetadata.schema.type.DataModelType", @@ -25,7 +28,8 @@ "QlikDataModel", "QuickSightDataModel", "SigmaDataModel", - "PowerBIDataFlow" + "PowerBIDataFlow", + "MicroStrategyDataset" ], "javaEnums": [ { @@ -63,6 +67,9 @@ }, { "name": "PowerBIDataFlow" + }, + { + "name": "MicroStrategyDataset" } ] } @@ -109,9 +116,9 @@ "$ref": "../../type/entityReferenceList.json", "default": null }, - "dataProducts" : { + "dataProducts": { "description": "List of data products this entity is part of.", - "$ref" : "../../type/entityReferenceList.json" + "$ref": "../../type/entityReferenceList.json" }, "tags": { "description": "Tags for this data model.", @@ -199,4 +206,4 @@ "columns" ], "additionalProperties": false -} +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts index c6bf09643d8..4e8658a2f00 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts @@ -733,6 +733,7 @@ export enum DataModelType { LookMlExplore = "LookMlExplore", LookMlView = "LookMlView", MetabaseDataModel = "MetabaseDataModel", + MicroStrategyDataset = "MicroStrategyDataset", PowerBIDataFlow = "PowerBIDataFlow", PowerBIDataModel = "PowerBIDataModel", QlikDataModel = "QlikDataModel",