fix(ingest/sigma): migrate sigma workbooks from container to dashboard (#11939)

2025-08-18 14:16:48 +00:00 · 2024-11-26 18:56:30 +05:30 · 2024-11-26 18:56:30 +05:30 · b357f87f94
commit b357f87f94
parent 3423e34899
6 changed files with 1523 additions and 1713 deletions
--- a/metadata-ingestion/docs/sources/sigma/sigma_pre.md
+++ b/metadata-ingestion/docs/sources/sigma/sigma_pre.md
@ -16,7 +16,7 @@ This source extracts the following:
 | Sigma                  | Datahub												         | Notes                            |
 |------------------------|---------------------------------------------------------------|----------------------------------|
 | `Workspace`            | [Container](../../metamodel/entities/container.md)     	     | SubType `"Sigma Workspace"`      |
-| `Workbook`             | [Container](../../metamodel/entities/container.md)            | SubType `"Sigma Workbook"`       |
+| `Workbook`             | [Dashboard](../../metamodel/entities/dashboard.md)            | SubType `"Sigma Workbook"`       |
 | `Page`                 | [Dashboard](../../metamodel/entities/dashboard.md)            |                                  |
 | `Element`              | [Chart](../../metamodel/entities/chart.md)                    |                                  |
 | `Dataset`              | [Dataset](../../metamodel/entities/dataset.md)                | SubType `"Sigma Dataset"`        |
--- a/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py
@ -80,6 +80,7 @@ class Workbook(BaseModel):
    path: str
    latestVersion: int
    workspaceId: Optional[str] = None
+    description: Optional[str] = None
    pages: List[Page] = []
    badge: Optional[str] = None

--- a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py
@ -4,7 +4,12 @@ from typing import Dict, Iterable, List, Optional
 import datahub.emitter.mce_builder as builder
 from datahub.configuration.common import ConfigurationError
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.emitter.mcp_builder import add_entity_to_container, gen_containers
+from datahub.emitter.mcp_builder import (
+    add_entity_to_container,
+    add_owner_to_entity_wu,
+    add_tags_to_entity_wu,
+    gen_containers,
+)
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
    SourceCapability,
@ -59,12 +64,14 @@ from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
    UpstreamLineage,
 )
 from datahub.metadata.schema_classes import (
+    AuditStampClass,
    BrowsePathEntryClass,
    BrowsePathsV2Class,
    ChangeAuditStampsClass,
    ChartInfoClass,
    DashboardInfoClass,
    DataPlatformInstanceClass,
+    EdgeClass,
    GlobalTagsClass,
    InputFieldClass,
    InputFieldsClass,
@ -74,6 +81,7 @@ from datahub.metadata.schema_classes import (
    SchemaFieldClass,
    SchemaFieldDataTypeClass,
    StringTypeClass,
+    SubTypesClass,
    TagAssociationClass,
 )
 from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result
@ -257,11 +265,6 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
        entries = [
            BrowsePathEntryClass(id=parent_entity_urn, urn=parent_entity_urn)
        ] + [BrowsePathEntryClass(id=path) for path in paths]
-        if self.config.platform_instance:
-            urn = builder.make_dataplatform_instance_urn(
-                self.platform, self.config.platform_instance
-            )
-            entries = [BrowsePathEntryClass(id=urn, urn=urn)] + entries
        return MetadataChangeProposalWrapper(
            entityUrn=entity_urn,
            aspect=BrowsePathsV2Class(entries),
@ -424,11 +427,11 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
        elements: List[Element],
        workbook: Workbook,
        all_input_fields: List[InputFieldClass],
+        paths: List[str],
    ) -> Iterable[MetadataWorkUnit]:
        """
        Map Sigma page element to Datahub Chart
        """
-
        for element in elements:
            chart_urn = builder.make_chart_urn(
                platform=self.platform,
@ -459,10 +462,13 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
                ),
            ).as_workunit()

-            yield from add_entity_to_container(
-                container_key=self._gen_workbook_key(workbook.workbookId),
-                entity_type="chart",
+            if workbook.workspaceId:
+                yield self._gen_entity_browsepath_aspect(
                    entity_urn=chart_urn,
+                    parent_entity_urn=builder.make_container_urn(
+                        self._gen_workspace_key(workbook.workspaceId)
+                    ),
+                    paths=paths + [workbook.name],
                )

            # Add sigma dataset's upstream dataset urn mapping
@ -494,7 +500,9 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):

            all_input_fields.extend(element_input_fields)

-    def _gen_pages_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]:
+    def _gen_pages_workunit(
+        self, workbook: Workbook, paths: List[str]
+    ) -> Iterable[MetadataWorkUnit]:
        """
        Map Sigma workbook page to Datahub dashboard
        """
@ -505,20 +513,23 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):

            yield self._gen_dashboard_info_workunit(page)

-            yield from add_entity_to_container(
-                container_key=self._gen_workbook_key(workbook.workbookId),
-                entity_type="dashboard",
-                entity_urn=dashboard_urn,
-            )
-
            dpi_aspect = self._gen_dataplatform_instance_aspect(dashboard_urn)
            if dpi_aspect:
                yield dpi_aspect

            all_input_fields: List[InputFieldClass] = []

+            if workbook.workspaceId:
+                yield self._gen_entity_browsepath_aspect(
+                    entity_urn=dashboard_urn,
+                    parent_entity_urn=builder.make_container_urn(
+                        self._gen_workspace_key(workbook.workspaceId)
+                    ),
+                    paths=paths + [workbook.name],
+                )
+
            yield from self._gen_elements_workunit(
-                page.elements, workbook, all_input_fields
+                page.elements, workbook, all_input_fields, paths
            )

            yield MetadataChangeProposalWrapper(
@ -531,42 +542,89 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
        Map Sigma Workbook to Datahub container
        """
        owner_username = self.sigma_api.get_user_name(workbook.createdBy)
-        workbook_key = self._gen_workbook_key(workbook.workbookId)
-        yield from gen_containers(
-            container_key=workbook_key,
-            name=workbook.name,
-            sub_types=[BIContainerSubTypes.SIGMA_WORKBOOK],
-            parent_container_key=(
-                self._gen_workspace_key(workbook.workspaceId)
-                if workbook.workspaceId
-                else None
+
+        dashboard_urn = self._gen_dashboard_urn(workbook.workbookId)
+
+        yield self._gen_entity_status_aspect(dashboard_urn)
+
+        lastModified = AuditStampClass(
+            time=int(workbook.updatedAt.timestamp() * 1000),
+            actor="urn:li:corpuser:datahub",
+        )
+        created = AuditStampClass(
+            time=int(workbook.createdAt.timestamp() * 1000),
+            actor="urn:li:corpuser:datahub",
+        )
+
+        dashboard_info_cls = DashboardInfoClass(
+            title=workbook.name,
+            description=workbook.description if workbook.description else "",
+            dashboards=[
+                EdgeClass(
+                    destinationUrn=self._gen_dashboard_urn(page.get_urn_part()),
+                    sourceUrn=dashboard_urn,
+                )
+                for page in workbook.pages
+            ],
+            externalUrl=workbook.url,
+            lastModified=ChangeAuditStampsClass(
+                created=created, lastModified=lastModified
            ),
-            extra_properties={
+            customProperties={
                "path": workbook.path,
                "latestVersion": str(workbook.latestVersion),
            },
-            owner_urn=(
+        )
+        yield MetadataChangeProposalWrapper(
+            entityUrn=dashboard_urn, aspect=dashboard_info_cls
+        ).as_workunit()
+
+        # Set subtype
+        yield MetadataChangeProposalWrapper(
+            entityUrn=dashboard_urn,
+            aspect=SubTypesClass(typeNames=[BIContainerSubTypes.SIGMA_WORKBOOK]),
+        ).as_workunit()
+
+        # Ownership
+        owner_urn = (
            builder.make_user_urn(owner_username)
            if self.config.ingest_owner and owner_username
            else None
-            ),
-            external_url=workbook.url,
-            tags=[workbook.badge] if workbook.badge else None,
-            created=int(workbook.createdAt.timestamp() * 1000),
-            last_modified=int(workbook.updatedAt.timestamp() * 1000),
+        )
+        if owner_urn:
+            yield from add_owner_to_entity_wu(
+                entity_type="dashboard",
+                entity_urn=dashboard_urn,
+                owner_urn=owner_urn,
+            )
+
+        # Tags
+        tags = [workbook.badge] if workbook.badge else None
+        if tags:
+            yield from add_tags_to_entity_wu(
+                entity_type="dashboard",
+                entity_urn=dashboard_urn,
+                tags=sorted(tags),
            )

        paths = workbook.path.split("/")[1:]
-        if len(paths) > 0 and workbook.workspaceId:
+        if workbook.workspaceId:
            yield self._gen_entity_browsepath_aspect(
-                entity_urn=builder.make_container_urn(workbook_key),
+                entity_urn=dashboard_urn,
                parent_entity_urn=builder.make_container_urn(
                    self._gen_workspace_key(workbook.workspaceId)
                ),
-                paths=paths,
+                paths=paths + [workbook.name],
            )

-        yield from self._gen_pages_workunit(workbook)
+            if len(paths) == 0:
+                yield from add_entity_to_container(
+                    container_key=self._gen_workspace_key(workbook.workspaceId),
+                    entity_type="dashboard",
+                    entity_urn=dashboard_urn,
+                )
+
+        yield from self._gen_pages_workunit(workbook, paths)

    def _gen_sigma_dataset_upstream_lineage_workunit(
        self,
--- a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json
+++ b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json
--- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json
+++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json
@ -261,37 +261,8 @@
    }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "containerProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "platform": "sigma",
-                "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b",
-                "path": "Acryl Data",
-                "latestVersion": "2"
-            },
-            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
-            "name": "Acryl Workbook",
-            "created": {
-                "time": 1713188691477
-            },
-            "lastModified": {
-                "time": 1713189117302
-            }
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619227,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
    "changeType": "UPSERT",
    "aspectName": "status",
    "aspect": {
@ -300,7 +271,7 @@
        }
    },
    "systemMetadata": {
-        "lastObserved": 1713795619228,
+        "lastObserved": 1732513099680,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
@ -337,6 +308,114 @@
        "lastRunId": "no-run-id-provided"
    }
 },
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Sigma Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732513099681,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "path": "Acryl Data",
+                "latestVersion": "2"
+            },
+            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
+            "title": "Acryl Workbook",
+            "description": "",
+            "charts": [],
+            "datasets": [],
+            "dashboards": [
+                {
+                    "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+                    "destinationUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)"
+                },
+                {
+                    "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+                    "destinationUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)"
+                }
+            ],
+            "lastModified": {
+                "created": {
+                    "time": 1713188691477,
+                    "actor": "urn:li:corpuser:datahub"
+                },
+                "lastModified": {
+                    "time": 1713189117302,
+                    "actor": "urn:li:corpuser:datahub"
+                }
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732535135915,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:Shubham_Jagtap",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732513099681,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": [
+                {
+                    "tag": "urn:li:tag:Warning"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732513099682,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
@ -361,66 +440,6 @@
        "lastRunId": "no-run-id-provided"
    }
 },
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "ownership",
-    "aspect": {
-        "json": {
-            "owners": [
-                {
-                    "owner": "urn:li:corpuser:Shubham_Jagtap",
-                    "type": "DATAOWNER"
-                }
-            ],
-            "ownerTypes": {},
-            "lastModified": {
-                "time": 0,
-                "actor": "urn:li:corpuser:unknown"
-            }
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619229,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "Sigma Workbook"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619229,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
-    "aspect": {
-        "json": {
-            "platform": "urn:li:dataPlatform:sigma"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619228,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
@ -453,6 +472,70 @@
        "lastRunId": "no-run-id-provided"
    }
 },
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
+                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
+                },
+                {
+                    "id": "Acryl Workbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732545848809,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732513099682,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
+                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
+                },
+                {
+                    "id": "Acryl Workbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732545848807,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
    "entityType": "dashboard",
    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
@ -470,6 +553,7 @@
                "urn:li:chart:(sigma,Ml9C5ezT5W)"
            ],
            "datasets": [],
+            "dashboards": [],
            "lastModified": {
                "created": {
                    "time": 0,
@ -514,42 +598,6 @@
        "lastRunId": "no-run-id-provided"
    }
 },
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "globalTags",
-    "aspect": {
-        "json": {
-            "tags": [
-                {
-                    "tag": "urn:li:tag:Warning"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619230,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619234,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
@ -570,47 +618,6 @@
        "lastRunId": "no-run-id-provided"
    }
 },
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
-                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
-                },
-                {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619234,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619231,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
@ -642,51 +649,13 @@
                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
                },
                {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                }
            ]
        }
    },
    "systemMetadata": {
-        "lastObserved": 1713795619375,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
-                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619231,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619373,
+        "lastObserved": 1732545848872,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
@ -883,6 +852,7 @@
                "urn:li:chart:(sigma,tQJu5N1l81)"
            ],
            "datasets": [],
+            "dashboards": [],
            "lastModified": {
                "created": {
                    "time": 0,
@ -901,6 +871,30 @@
        "lastRunId": "no-run-id-provided"
    }
 },
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
+                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
+                },
+                {
+                    "id": "Acryl Workbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732545848877,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
    "entityType": "chart",
    "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
@ -914,14 +908,13 @@
                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
                },
                {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                }
            ]
        }
    },
    "systemMetadata": {
-        "lastObserved": 1713795619270,
+        "lastObserved": 1732545848829,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
@ -1179,54 +1172,6 @@
        "lastRunId": "no-run-id-provided"
    }
 },
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619382,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619267,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619449,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
    "entityType": "chart",
    "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
@ -1409,31 +1354,6 @@
        "lastRunId": "no-run-id-provided"
    }
 },
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
-                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
-                },
-                {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619383,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
    "entityType": "chart",
    "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
@ -1554,14 +1474,13 @@
                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
                },
                {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                }
            ]
        }
    },
    "systemMetadata": {
-        "lastObserved": 1713795619453,
+        "lastObserved": 1732545848921,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
--- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json
+++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json
@ -279,37 +279,8 @@
    }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "containerProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "platform": "sigma",
-                "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b",
-                "path": "New Acryl Data",
-                "latestVersion": "2"
-            },
-            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
-            "name": "Acryl Workbook",
-            "created": {
-                "time": 1713188691477
-            },
-            "lastModified": {
-                "time": 1713189117302
-            }
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101680,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
    "changeType": "UPSERT",
    "aspectName": "status",
    "aspect": {
@ -318,30 +289,30 @@
        }
    },
    "systemMetadata": {
-        "lastObserved": 1718004101680,
+        "lastObserved": 1732513100094,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
    "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
+    "aspectName": "status",
    "aspect": {
        "json": {
-            "platform": "urn:li:dataPlatform:sigma"
+            "removed": false
        }
    },
    "systemMetadata": {
-        "lastObserved": 1718004101681,
+        "lastObserved": 1718004101684,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
    "changeType": "UPSERT",
    "aspectName": "subTypes",
    "aspect": {
@ -352,14 +323,58 @@
        }
    },
    "systemMetadata": {
-        "lastObserved": 1718004101681,
+        "lastObserved": 1732513100095,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "path": "New Acryl Data",
+                "latestVersion": "2"
+            },
+            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
+            "title": "Acryl Workbook",
+            "description": "",
+            "charts": [],
+            "datasets": [],
+            "dashboards": [
+                {
+                    "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+                    "destinationUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)"
+                },
+                {
+                    "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+                    "destinationUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)"
+                }
+            ],
+            "lastModified": {
+                "created": {
+                    "time": 1713188691477,
+                    "actor": "urn:li:corpuser:datahub"
+                },
+                "lastModified": {
+                    "time": 1713189117302,
+                    "actor": "urn:li:corpuser:datahub"
+                }
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732535136409,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
    "changeType": "UPSERT",
    "aspectName": "ownership",
    "aspect": {
@ -378,14 +393,14 @@
        }
    },
    "systemMetadata": {
-        "lastObserved": 1718004101682,
+        "lastObserved": 1732513100096,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
    "changeType": "UPSERT",
    "aspectName": "globalTags",
    "aspect": {
@ -398,60 +413,7 @@
        }
    },
    "systemMetadata": {
-        "lastObserved": 1718004101683,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:b83da80a4d444484521d9f7aca958742"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101683,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b83da80a4d444484521d9f7aca958742",
-                    "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101684,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101684,
+        "lastObserved": 1732513100096,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
@ -473,6 +435,7 @@
                "urn:li:chart:(sigma,Ml9C5ezT5W)"
            ],
            "datasets": [],
+            "dashboards": [],
            "lastModified": {
                "created": {
                    "time": 0,
@ -491,47 +454,6 @@
        "lastRunId": "no-run-id-provided"
    }
 },
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101686,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b83da80a4d444484521d9f7aca958742",
-                    "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
-                },
-                {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101686,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
    "entityType": "chart",
    "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
@ -582,17 +504,65 @@
    }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b83da80a4d444484521d9f7aca958742",
+                    "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
+                },
+                {
+                    "id": "Acryl Workbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732545849249,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
    "changeType": "UPSERT",
    "aspectName": "container",
    "aspect": {
        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+            "container": "urn:li:container:b83da80a4d444484521d9f7aca958742"
        }
    },
    "systemMetadata": {
-        "lastObserved": 1718004101689,
+        "lastObserved": 1732513100096,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b83da80a4d444484521d9f7aca958742",
+                    "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
+                },
+                {
+                    "id": "Acryl Workbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732545849248,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
@ -717,14 +687,13 @@
                    "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
                },
                {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                }
            ]
        }
    },
    "systemMetadata": {
-        "lastObserved": 1718004101692,
+        "lastObserved": 1732545849252,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
@ -778,22 +747,6 @@
        "lastRunId": "no-run-id-provided"
    }
 },
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101695,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
    "entityType": "chart",
    "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)",
@ -914,14 +867,13 @@
                    "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
                },
                {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                }
            ]
        }
    },
    "systemMetadata": {
-        "lastObserved": 1718004101697,
+        "lastObserved": 1732545849255,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
@ -1155,6 +1107,7 @@
                "urn:li:chart:(sigma,tQJu5N1l81)"
            ],
            "datasets": [],
+            "dashboards": [],
            "lastModified": {
                "created": {
                    "time": 0,
@ -1174,17 +1127,17 @@
    }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
    "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "status",
    "aspect": {
        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+            "removed": false
        }
    },
    "systemMetadata": {
-        "lastObserved": 1718004101704,
+        "lastObserved": 1718004101706,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
@ -1202,30 +1155,13 @@
                    "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
                },
                {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                }
            ]
        }
    },
    "systemMetadata": {
-        "lastObserved": 1718004101704,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101706,
+        "lastObserved": 1732545849260,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }
@ -1263,22 +1199,6 @@
        "lastRunId": "no-run-id-provided"
    }
 },
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101708,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
    "entityType": "chart",
    "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
@ -1474,14 +1394,13 @@
                    "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
                },
                {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                }
            ]
        }
    },
    "systemMetadata": {
-        "lastObserved": 1718004101712,
+        "lastObserved": 1732545849264,
        "runId": "sigma-test",
        "lastRunId": "no-run-id-provided"
    }