Feature: Microstrategy Lineage (#21678)

ingestion/src/metadata/ingestion/source/dashboard/microstrategy/client.py

@@ -266,3 +266,24 @@ class MicroStrategyClient:
             logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}")
 
         return None
+
+    def get_cube_sql_details(self, project_id: str, cube_id: str) -> Optional[str]:
+        """
+        Get Cube SQL Details
+        """
+        try:
+            headers = {
+                "X-MSTR-ProjectID": project_id,
+                "cubeId": cube_id,
+            } | self.auth_params.auth_header
+
+            resp_dataset = self.client._request(  # pylint: disable=protected-access
+                "GET", path=f"/v2/cubes/{cube_id}/sqlView", headers=headers
+            )
+            return resp_dataset["sqlStatement"]
+
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch the cube with id: {cube_id}")
+
+        return None

ingestion/src/metadata/ingestion/source/dashboard/microstrategy/helpers.py

@@ -0,0 +1,67 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Microstrategy source helpers.
+"""
+from __future__ import annotations
+
+from typing import Any, Dict
+
+from metadata.generated.schema.entity.data.table import Column, DataType
+
+
+class MicroStrategyColumnParser:
+    """
+    Responsible for containing the logic to parse a column from MicroStrategy to OpenMetadata
+    """
+
+    datatype_mapping = {
+        "big decimal": DataType.DECIMAL,
+        "binary": DataType.BYTES,
+        "char": DataType.CHAR,
+        "date": DataType.DATE,
+        "decimal": DataType.DECIMAL,
+        "double": DataType.DOUBLE,
+        "float": DataType.FLOAT,
+        "integer": DataType.INT,
+        "longvarbin": DataType.BLOB,
+        "longvarchar": DataType.CLOB,
+        "nchar": DataType.CHAR,
+        "numeric": DataType.NUMERIC,
+        "nvarchar": DataType.VARCHAR,
+        "real": DataType.STRING,
+        "time": DataType.TIME,
+        "timestamp": DataType.TIMESTAMP,
+        "unsigned": DataType.INT,
+        "varbin": DataType.BYTES,
+        "varchar": DataType.VARCHAR,
+        "utf8char": DataType.CHAR,
+    }
+
+    @classmethod
+    def parse(cls, field: Dict[str, Any]) -> Column:
+        """
+        Parses a MicroStrategy table column into an OpenMetadata column.
+        """
+
+        array_data_type = None
+        data_type = cls.datatype_mapping.get(
+            field["dataType"].lower(), DataType.UNKNOWN
+        )
+
+        column_def = {
+            "name": field["name"],
+            "dataTypeDisplay": field["dataType"],
+            "dataType": data_type,
+            "arrayDataType": array_data_type,
+        }
+
+        return Column(**column_def)

ingestion/src/metadata/ingestion/source/dashboard/microstrategy/metadata.py

@@ -14,11 +14,23 @@ from typing import Iterable, List, Optional
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
+from metadata.generated.schema.api.data.createDashboardDataModel import (
+    CreateDashboardDataModelRequest,
+)
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.chart import Chart
+from metadata.generated.schema.entity.data.dashboardDataModel import (
+    DashboardDataModel,
+    DataModelType,
+)
+from metadata.generated.schema.entity.data.table import Column, DataType
 from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
     MicroStrategyConnection,
 )
+from metadata.generated.schema.entity.services.dashboardService import (
+    DashboardServiceType,
+)
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
@@ -29,18 +41,29 @@ from metadata.generated.schema.type.basic import (
     EntityName,
     FullyQualifiedEntityName,
     SourceUrl,
+    Uuid,
 )
+from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
+from metadata.generated.schema.type.entityLineage import Source as LineageSource
+from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
+from metadata.ingestion.lineage.parser import LineageParser
+from metadata.ingestion.lineage.sql_lineage import get_table_entities_from_query
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
+from metadata.ingestion.source.dashboard.microstrategy.helpers import (
+    MicroStrategyColumnParser,
+)
 from metadata.ingestion.source.dashboard.microstrategy.models import (
     MstrDashboard,
     MstrDashboardDetails,
+    MstrDataset,
     MstrPage,
 )
 from metadata.utils import fqn
-from metadata.utils.filters import filter_by_chart
+from metadata.utils.filters import filter_by_chart, filter_by_datamodel
 from metadata.utils.helpers import clean_uri, get_standard_chart_type
 from metadata.utils.logger import ingestion_logger
 
@@ -163,7 +186,75 @@ class MicrostrategySource(DashboardServiceSource):
         dashboard_details: MstrDashboardDetails,
         db_service_name: Optional[str] = None,
     ) -> Optional[Iterable[AddLineageRequest]]:
-        """Not Implemented"""
+        """
+        Get lineage between datamodel and data sources
+        """
+        if not db_service_name:
+            return
+
+        database_service = self.metadata.get_by_name(
+            entity=DatabaseService, fqn=db_service_name
+        )
+        dialect = ConnectionTypeDialectMapper.dialect_of(
+            database_service.serviceType.value
+        )
+
+        for dataset in dashboard_details.datasets:
+            cube_sql = self.client.get_cube_sql_details(
+                dashboard_details.projectId, dataset.id
+            )
+            if not cube_sql:
+                continue
+
+            datamodel_fqn = fqn.build(
+                self.metadata,
+                entity_type=DashboardDataModel,
+                service_name=self.context.get().dashboard_service,
+                data_model_name=dataset.id,
+            )
+            datamodel_entity = self.metadata.get_by_name(
+                entity=DashboardDataModel, fqn=datamodel_fqn
+            )
+
+            try:
+                lineage_parser = LineageParser(cube_sql, dialect=dialect)
+                for table in lineage_parser.source_tables:
+                    table_entities = get_table_entities_from_query(
+                        metadata=self.metadata,
+                        service_name=db_service_name,
+                        database_name="*",
+                        database_schema="*",
+                        table_name=str(table),
+                    )
+                    if not table_entities:
+                        logger.debug(f"Table not found in metadata: {str(table)}")
+                        continue
+                    for table_entity in table_entities or []:
+                        yield Either(
+                            right=AddLineageRequest(
+                                edge=EntitiesEdge(
+                                    fromEntity=EntityReference(
+                                        id=Uuid(table_entity.id.root),
+                                        type="table",
+                                    ),
+                                    toEntity=EntityReference(
+                                        id=Uuid(datamodel_entity.id.root),
+                                        type="dashboardDataModel",
+                                    ),
+                                    lineageDetails=LineageDetails(
+                                        source=LineageSource.DashboardLineage
+                                    ),
+                                )
+                            )
+                        )
+            except Exception as exc:
+                yield Either(
+                    left=StackTraceError(
+                        name="Dashboard Lineage",
+                        error=f"Error to yield dashboard lineage details: {exc}",
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
 
     def yield_dashboard_chart(
         self, dashboard_details: MstrDashboardDetails
@@ -213,6 +304,74 @@ class MicrostrategySource(DashboardServiceSource):
                     )
                 )
 
+    def _get_column_info(self, dataset: MstrDataset) -> Optional[List[Column]]:
+        """Build columns from dataset"""
+        datasource_columns = []
+        for available_object in dataset.availableObjects or []:
+            try:
+                parsed_column = {
+                    "dataTypeDisplay": available_object.type.title(),
+                    "dataType": DataType.UNKNOWN,
+                    "name": available_object.name,
+                    "displayName": available_object.name,
+                }
+                parsed_column_children = []
+                for form in available_object.forms or []:
+                    parsed_column_children.append(MicroStrategyColumnParser.parse(form))
+                if parsed_column_children:
+                    parsed_column["children"] = parsed_column_children
+
+                datasource_columns.append(Column(**parsed_column))
+            except Exception as exc:
+                logger.debug(traceback.format_exc())
+                logger.warning(f"Error to yield datamodel column: {exc}")
+        return datasource_columns
+
+    def yield_datamodel(
+        self, dashboard_details: MstrDashboardDetails
+    ) -> Optional[Iterable[CreateDashboardDataModelRequest]]:
+        """Get datamodel method
+
+        Args:
+            dashboard_details:
+        Returns:
+            Iterable[CreateDashboardDataModelRequest]
+        """
+        try:
+            if self.source_config.includeDataModels:
+                for dataset in dashboard_details.datasets:
+                    if filter_by_datamodel(
+                        self.source_config.dataModelFilterPattern, dataset.name
+                    ):
+                        self.status.filter(dataset.name, "Data model filtered out.")
+                        continue
+                    data_model_type = DataModelType.MicroStrategyDataset.value
+                    datamodel_columns = self._get_column_info(dataset)
+
+                    data_model_request = CreateDashboardDataModelRequest(
+                        name=EntityName(dataset.id),
+                        displayName=dataset.name,
+                        service=FullyQualifiedEntityName(
+                            self.context.get().dashboard_service
+                        ),
+                        dataModelType=data_model_type,
+                        serviceType=DashboardServiceType.MicroStrategy.value,
+                        columns=datamodel_columns,
+                        project=self.get_project_name(
+                            dashboard_details=dashboard_details
+                        ),
+                    )
+                    yield Either(right=data_model_request)
+                    self.register_record_datamodel(datamodel_request=data_model_request)
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=dataset.name,
+                    error=f"Error yielding Data Model [{dataset.name}]: {exc}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )
+
     def close(self):
         # close the api session
         self.client.close_api_session()

ingestion/src/metadata/ingestion/source/dashboard/microstrategy/models.py

@@ -12,7 +12,7 @@
 MicroStrategy Models
 """
 from datetime import datetime
-from typing import Any, List, Optional
+from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel
 
@@ -134,6 +134,17 @@ class MstrAvailableObject(BaseModel):
     id: str
     name: str
     type: str
+    forms: Optional[List[Dict[str, Any]]] = None
+
+
+class MstrDataset(BaseModel):
+    id: str
+    name: str
+    availableObjects: Optional[List[MstrAvailableObject]] = None
+    rows: Optional[List[Dict[str, Any]]] = None
+    columns: Optional[List[Dict[str, Any]]] = None
+    pageBy: Optional[List[Dict[str, Any]]] = None
+    sqlStatement: Optional[str] = None
 
 
 class MstrDashboardDetails(BaseModel):
@@ -143,6 +154,7 @@ class MstrDashboardDetails(BaseModel):
     projectName: str
     currentChapter: str
     chapters: List[MstrChapter]
+    datasets: List[MstrDataset]
 
 
 class AuthHeaderCookie(BaseModel):

openmetadata-docs/content/v1.7.x/connectors/dashboard/microstrategy/index.md

@@ -7,8 +7,8 @@ slug: /connectors/dashboard/microstrategy
   name="MicroStrategy"
   stage="PROD"
   platform="OpenMetadata"
-  availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels"]
-  unavailableFeatures=["Tags", "Projects", "Lineage"]
+  availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels", "Lineage"]
+  unavailableFeatures=["Tags", "Projects"]
 / %}
 
 In this section, we provide guides and references to use the MicroStrategy connector.

openmetadata-docs/content/v1.7.x/connectors/dashboard/microstrategy/yaml.md

@@ -7,8 +7,8 @@ slug: /connectors/dashboard/microstrategy/yaml
   name="MicroStrategy"
   stage="PROD"
   platform="OpenMetadata"
-  availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels"]
-  unavailableFeatures=["Tags", "Projects", "Lineage"]
+  availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels", "Lineage"]
+  unavailableFeatures=["Tags", "Projects"]
 / %}
 
 In this section, we provide guides and references to use the MicroStrategy connector.

openmetadata-docs/content/v1.8.x-SNAPSHOT/connectors/dashboard/microstrategy/index.md

@@ -7,8 +7,8 @@ slug: /connectors/dashboard/microstrategy
   name="MicroStrategy"
   stage="PROD"
   platform="OpenMetadata"
-  availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels"]
-  unavailableFeatures=["Tags", "Projects", "Lineage"]
+  availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels", "Lineage"]
+  unavailableFeatures=["Tags", "Projects"]
 / %}
 
 In this section, we provide guides and references to use the MicroStrategy connector.

openmetadata-docs/content/v1.8.x-SNAPSHOT/connectors/dashboard/microstrategy/yaml.md

@@ -7,8 +7,8 @@ slug: /connectors/dashboard/microstrategy/yaml
   name="MicroStrategy"
   stage="PROD"
   platform="OpenMetadata"
-  availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels"]
-  unavailableFeatures=["Tags", "Projects", "Lineage"]
+  availableFeatures=["Dashboards", "Charts", "Owners", "Datamodels", "Lineage"]
+  unavailableFeatures=["Tags", "Projects"]
 / %}
 
 In this section, we provide guides and references to use the MicroStrategy connector.

openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json

@@ -6,7 +6,10 @@
   "description": "Dashboard Data Model entity definition. Data models are the schemas used to build dashboards, charts, or other data assets.",
   "type": "object",
   "javaType": "org.openmetadata.schema.entity.data.DashboardDataModel",
-  "javaInterfaces": ["org.openmetadata.schema.EntityInterface", "org.openmetadata.schema.ColumnsEntityInterface"],
+  "javaInterfaces": [
+    "org.openmetadata.schema.EntityInterface",
+    "org.openmetadata.schema.ColumnsEntityInterface"
+  ],
   "definitions": {
     "dataModelType": {
       "javaType": "org.openmetadata.schema.type.DataModelType",
@@ -25,7 +28,8 @@
         "QlikDataModel",
         "QuickSightDataModel",
         "SigmaDataModel",
-        "PowerBIDataFlow"
+        "PowerBIDataFlow",
+        "MicroStrategyDataset"
       ],
       "javaEnums": [
         {
@@ -63,6 +67,9 @@
         },
         {
           "name": "PowerBIDataFlow"
+        },
+        {
+          "name": "MicroStrategyDataset"
         }
       ]
     }
@@ -109,9 +116,9 @@
       "$ref": "../../type/entityReferenceList.json",
       "default": null
     },
-    "dataProducts" : {
+    "dataProducts": {
       "description": "List of data products this entity is part of.",
-      "$ref" : "../../type/entityReferenceList.json"
+      "$ref": "../../type/entityReferenceList.json"
     },
     "tags": {
       "description": "Tags for this data model.",
@@ -199,4 +206,4 @@
     "columns"
   ],
   "additionalProperties": false
-}
+}

openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts

@@ -733,6 +733,7 @@ export enum DataModelType {
     LookMlExplore = "LookMlExplore",
     LookMlView = "LookMlView",
     MetabaseDataModel = "MetabaseDataModel",
+    MicroStrategyDataset = "MicroStrategyDataset",
     PowerBIDataFlow = "PowerBIDataFlow",
     PowerBIDataModel = "PowerBIDataModel",
     QlikDataModel = "QlikDataModel",