yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-18 14:16:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/sigma): migrate sigma workbooks from container to dashboard (#11939)

Browse Source
This commit is contained in:
sagar-salvi-apptware 2024-11-26 18:56:30 +05:30 committed by GitHub
parent 3423e34899
commit b357f87f94
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 1523 additions and 1713 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-ingestion/docs/sources/sigma/sigma_pre.md
View File

@ -16,7 +16,7 @@ This source extracts the following:
| Sigma | Datahub | Notes |
|------------------------|---------------------------------------------------------------|----------------------------------|
| `Workspace` | [Container](../../metamodel/entities/container.md) | SubType `"Sigma Workspace"` |
| `Workbook` | [Container](../../metamodel/entities/container.md) | SubType `"Sigma Workbook"` |
| `Workbook` | [Dashboard](../../metamodel/entities/dashboard.md) | SubType `"Sigma Workbook"` |
| `Page` | [Dashboard](../../metamodel/entities/dashboard.md) | |
| `Element` | [Chart](../../metamodel/entities/chart.md) | |
| `Dataset` | [Dataset](../../metamodel/entities/dataset.md) | SubType `"Sigma Dataset"` |

1
metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py
View File

@ -80,6 +80,7 @@ class Workbook(BaseModel):
path: str
latestVersion: int
workspaceId: Optional[str] = None
description: Optional[str] = None
pages: List[Page] = []
badge: Optional[str] = None

132
metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py
View File

@ -4,7 +4,12 @@ from typing import Dict, Iterable, List, Optional
import datahub.emitter.mce_builder as builder
from datahub.configuration.common import ConfigurationError
from datahub.emitter.mcp import MetadataChangeProposalWrapper
from datahub.emitter.mcp_builder import add_entity_to_container, gen_containers
from datahub.emitter.mcp_builder import (
add_entity_to_container,
add_owner_to_entity_wu,
add_tags_to_entity_wu,
gen_containers,
)
from datahub.ingestion.api.common import PipelineContext
from datahub.ingestion.api.decorators import (
SourceCapability,
@ -59,12 +64,14 @@ from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
UpstreamLineage,
)
from datahub.metadata.schema_classes import (
AuditStampClass,
BrowsePathEntryClass,
BrowsePathsV2Class,
ChangeAuditStampsClass,
ChartInfoClass,
DashboardInfoClass,
DataPlatformInstanceClass,
EdgeClass,
GlobalTagsClass,
InputFieldClass,
InputFieldsClass,
@ -74,6 +81,7 @@ from datahub.metadata.schema_classes import (
SchemaFieldClass,
SchemaFieldDataTypeClass,
StringTypeClass,
SubTypesClass,
TagAssociationClass,
)
from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result
@ -257,11 +265,6 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
entries = [
BrowsePathEntryClass(id=parent_entity_urn, urn=parent_entity_urn)
] + [BrowsePathEntryClass(id=path) for path in paths]
if self.config.platform_instance:
urn = builder.make_dataplatform_instance_urn(
self.platform, self.config.platform_instance
)
entries = [BrowsePathEntryClass(id=urn, urn=urn)] + entries
return MetadataChangeProposalWrapper(
entityUrn=entity_urn,
aspect=BrowsePathsV2Class(entries),
@ -424,11 +427,11 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
elements: List[Element],
workbook: Workbook,
all_input_fields: List[InputFieldClass],
paths: List[str],
) -> Iterable[MetadataWorkUnit]:
"""
Map Sigma page element to Datahub Chart
"""
for element in elements:
chart_urn = builder.make_chart_urn(
platform=self.platform,
@ -459,10 +462,13 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
),
).as_workunit()
yield from add_entity_to_container(
container_key=self._gen_workbook_key(workbook.workbookId),
entity_type="chart",
if workbook.workspaceId:
yield self._gen_entity_browsepath_aspect(
entity_urn=chart_urn,
parent_entity_urn=builder.make_container_urn(
self._gen_workspace_key(workbook.workspaceId)
),
paths=paths + [workbook.name],
)
# Add sigma dataset's upstream dataset urn mapping
@ -494,7 +500,9 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
all_input_fields.extend(element_input_fields)
def _gen_pages_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]:
def _gen_pages_workunit(
self, workbook: Workbook, paths: List[str]
) -> Iterable[MetadataWorkUnit]:
"""
Map Sigma workbook page to Datahub dashboard
"""
@ -505,20 +513,23 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
yield self._gen_dashboard_info_workunit(page)
yield from add_entity_to_container(
container_key=self._gen_workbook_key(workbook.workbookId),
entity_type="dashboard",
entity_urn=dashboard_urn,
)
dpi_aspect = self._gen_dataplatform_instance_aspect(dashboard_urn)
if dpi_aspect:
yield dpi_aspect
all_input_fields: List[InputFieldClass] = []
if workbook.workspaceId:
yield self._gen_entity_browsepath_aspect(
entity_urn=dashboard_urn,
parent_entity_urn=builder.make_container_urn(
self._gen_workspace_key(workbook.workspaceId)
),
paths=paths + [workbook.name],
)
yield from self._gen_elements_workunit(
page.elements, workbook, all_input_fields
page.elements, workbook, all_input_fields, paths
)
yield MetadataChangeProposalWrapper(
@ -531,42 +542,89 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
Map Sigma Workbook to Datahub container
"""
owner_username = self.sigma_api.get_user_name(workbook.createdBy)
workbook_key = self._gen_workbook_key(workbook.workbookId)
yield from gen_containers(
container_key=workbook_key,
name=workbook.name,
sub_types=[BIContainerSubTypes.SIGMA_WORKBOOK],
parent_container_key=(
self._gen_workspace_key(workbook.workspaceId)
if workbook.workspaceId
else None
dashboard_urn = self._gen_dashboard_urn(workbook.workbookId)
yield self._gen_entity_status_aspect(dashboard_urn)
lastModified = AuditStampClass(
time=int(workbook.updatedAt.timestamp() * 1000),
actor="urn:li:corpuser:datahub",
)
created = AuditStampClass(
time=int(workbook.createdAt.timestamp() * 1000),
actor="urn:li:corpuser:datahub",
)
dashboard_info_cls = DashboardInfoClass(
title=workbook.name,
description=workbook.description if workbook.description else "",
dashboards=[
EdgeClass(
destinationUrn=self._gen_dashboard_urn(page.get_urn_part()),
sourceUrn=dashboard_urn,
)
for page in workbook.pages
],
externalUrl=workbook.url,
lastModified=ChangeAuditStampsClass(
created=created, lastModified=lastModified
),
extra_properties={
customProperties={
"path": workbook.path,
"latestVersion": str(workbook.latestVersion),
},
)
yield MetadataChangeProposalWrapper(
entityUrn=dashboard_urn, aspect=dashboard_info_cls
).as_workunit()
# Set subtype
yield MetadataChangeProposalWrapper(
entityUrn=dashboard_urn,
aspect=SubTypesClass(typeNames=[BIContainerSubTypes.SIGMA_WORKBOOK]),
).as_workunit()
# Ownership
owner_urn = (
builder.make_user_urn(owner_username)
if self.config.ingest_owner and owner_username
else None
),
external_url=workbook.url,
tags=[workbook.badge] if workbook.badge else None,
created=int(workbook.createdAt.timestamp() * 1000),
last_modified=int(workbook.updatedAt.timestamp() * 1000),
)
if owner_urn:
yield from add_owner_to_entity_wu(
entity_type="dashboard",
entity_urn=dashboard_urn,
owner_urn=owner_urn,
)
# Tags
tags = [workbook.badge] if workbook.badge else None
if tags:
yield from add_tags_to_entity_wu(
entity_type="dashboard",
entity_urn=dashboard_urn,
tags=sorted(tags),
)
paths = workbook.path.split("/")[1:]
if len(paths) > 0 and workbook.workspaceId:
if workbook.workspaceId:
yield self._gen_entity_browsepath_aspect(
entity_urn=builder.make_container_urn(workbook_key),
entity_urn=dashboard_urn,
parent_entity_urn=builder.make_container_urn(
self._gen_workspace_key(workbook.workspaceId)
),
paths=paths,
paths=paths + [workbook.name],
)
yield from self._gen_pages_workunit(workbook)
if len(paths) == 0:
yield from add_entity_to_container(
container_key=self._gen_workspace_key(workbook.workspaceId),
entity_type="dashboard",
entity_urn=dashboard_urn,
)
yield from self._gen_pages_workunit(workbook, paths)
def _gen_sigma_dataset_upstream_lineage_workunit(
self,

2269
metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json
View File

File diff suppressed because it is too large Load Diff

495
metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json
View File

@ -261,37 +261,8 @@
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "sigma",
"workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b",
"path": "Acryl Data",
"latestVersion": "2"
},
"externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
"name": "Acryl Workbook",
"created": {
"time": 1713188691477
},
"lastModified": {
"time": 1713189117302
}
}
},
"systemMetadata": {
"lastObserved": 1713795619227,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
@ -300,7 +271,7 @@
}
},
"systemMetadata": {
"lastObserved": 1713795619228,
"lastObserved": 1732513099680,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
@ -337,6 +308,114 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Sigma Workbook"
]
}
},
"systemMetadata": {
"lastObserved": 1732513099681,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "dashboardInfo",
"aspect": {
"json": {
"customProperties": {
"path": "Acryl Data",
"latestVersion": "2"
},
"externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
"title": "Acryl Workbook",
"description": "",
"charts": [],
"datasets": [],
"dashboards": [
{
"sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"destinationUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)"
},
{
"sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"destinationUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)"
}
],
"lastModified": {
"created": {
"time": 1713188691477,
"actor": "urn:li:corpuser:datahub"
},
"lastModified": {
"time": 1713189117302,
"actor": "urn:li:corpuser:datahub"
}
}
}
},
"systemMetadata": {
"lastObserved": 1732535135915,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "ownership",
"aspect": {
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham_Jagtap",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
}
}
},
"systemMetadata": {
"lastObserved": 1732513099681,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "globalTags",
"aspect": {
"json": {
"tags": [
{
"tag": "urn:li:tag:Warning"
}
]
}
},
"systemMetadata": {
"lastObserved": 1732513099682,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
@ -361,66 +440,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"changeType": "UPSERT",
"aspectName": "ownership",
"aspect": {
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham_Jagtap",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
}
}
},
"systemMetadata": {
"lastObserved": 1713795619229,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Sigma Workbook"
]
}
},
"systemMetadata": {
"lastObserved": 1713795619229,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:sigma"
}
},
"systemMetadata": {
"lastObserved": 1713795619228,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
@ -453,6 +472,70 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
"urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
},
{
"id": "Acryl Workbook"
}
]
}
},
"systemMetadata": {
"lastObserved": 1732545848809,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
}
},
"systemMetadata": {
"lastObserved": 1732513099682,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
"urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
},
{
"id": "Acryl Workbook"
}
]
}
},
"systemMetadata": {
"lastObserved": 1732545848807,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
@ -470,6 +553,7 @@
"urn:li:chart:(sigma,Ml9C5ezT5W)"
],
"datasets": [],
"dashboards": [],
"lastModified": {
"created": {
"time": 0,
@ -514,42 +598,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"changeType": "UPSERT",
"aspectName": "globalTags",
"aspect": {
"json": {
"tags": [
{
"tag": "urn:li:tag:Warning"
}
]
}
},
"systemMetadata": {
"lastObserved": 1713795619230,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
}
},
"systemMetadata": {
"lastObserved": 1713795619234,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
@ -570,47 +618,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
"urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
},
{
"id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
}
]
}
},
"systemMetadata": {
"lastObserved": 1713795619234,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
}
},
"systemMetadata": {
"lastObserved": 1713795619231,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
@ -642,51 +649,13 @@
"urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
},
{
"id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
"id": "Acryl Workbook"
}
]
}
},
"systemMetadata": {
"lastObserved": 1713795619375,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
"urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
}
]
}
},
"systemMetadata": {
"lastObserved": 1713795619231,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
}
},
"systemMetadata": {
"lastObserved": 1713795619373,
"lastObserved": 1732545848872,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
@ -883,6 +852,7 @@
"urn:li:chart:(sigma,tQJu5N1l81)"
],
"datasets": [],
"dashboards": [],
"lastModified": {
"created": {
"time": 0,
@ -901,6 +871,30 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
"urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
},
{
"id": "Acryl Workbook"
}
]
}
},
"systemMetadata": {
"lastObserved": 1732545848877,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
@ -914,14 +908,13 @@
"urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
},
{
"id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
"id": "Acryl Workbook"
}
]
}
},
"systemMetadata": {
"lastObserved": 1713795619270,
"lastObserved": 1732545848829,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
@ -1179,54 +1172,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
}
},
"systemMetadata": {
"lastObserved": 1713795619382,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
}
},
"systemMetadata": {
"lastObserved": 1713795619267,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
}
},
"systemMetadata": {
"lastObserved": 1713795619449,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
@ -1409,31 +1354,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
"urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
},
{
"id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
}
]
}
},
"systemMetadata": {
"lastObserved": 1713795619383,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
@ -1554,14 +1474,13 @@
"urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
},
{
"id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
"id": "Acryl Workbook"
}
]
}
},
"systemMetadata": {
"lastObserved": 1713795619453,
"lastObserved": 1732545848921,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}

337
metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json
View File

@ -279,37 +279,8 @@
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "sigma",
"workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b",
"path": "New Acryl Data",
"latestVersion": "2"
},
"externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
"name": "Acryl Workbook",
"created": {
"time": 1713188691477
},
"lastModified": {
"time": 1713189117302
}
}
},
"systemMetadata": {
"lastObserved": 1718004101680,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
@ -318,30 +289,30 @@
}
},
"systemMetadata": {
"lastObserved": 1718004101680,
"lastObserved": 1732513100094,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspectName": "status",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:sigma"
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1718004101681,
"lastObserved": 1718004101684,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
@ -352,14 +323,58 @@
}
},
"systemMetadata": {
"lastObserved": 1718004101681,
"lastObserved": 1732513100095,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "dashboardInfo",
"aspect": {
"json": {
"customProperties": {
"path": "New Acryl Data",
"latestVersion": "2"
},
"externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
"title": "Acryl Workbook",
"description": "",
"charts": [],
"datasets": [],
"dashboards": [
{
"sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"destinationUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)"
},
{
"sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"destinationUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)"
}
],
"lastModified": {
"created": {
"time": 1713188691477,
"actor": "urn:li:corpuser:datahub"
},
"lastModified": {
"time": 1713189117302,
"actor": "urn:li:corpuser:datahub"
}
}
}
},
"systemMetadata": {
"lastObserved": 1732535136409,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "ownership",
"aspect": {
@ -378,14 +393,14 @@
}
},
"systemMetadata": {
"lastObserved": 1718004101682,
"lastObserved": 1732513100096,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "globalTags",
"aspect": {
@ -398,60 +413,7 @@
}
},
"systemMetadata": {
"lastObserved": 1718004101683,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:b83da80a4d444484521d9f7aca958742"
}
},
"systemMetadata": {
"lastObserved": 1718004101683,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:b83da80a4d444484521d9f7aca958742",
"urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
}
]
}
},
"systemMetadata": {
"lastObserved": 1718004101684,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1718004101684,
"lastObserved": 1732513100096,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
@ -473,6 +435,7 @@
"urn:li:chart:(sigma,Ml9C5ezT5W)"
],
"datasets": [],
"dashboards": [],
"lastModified": {
"created": {
"time": 0,
@ -491,47 +454,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
}
},
"systemMetadata": {
"lastObserved": 1718004101686,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:b83da80a4d444484521d9f7aca958742",
"urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
},
{
"id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
}
]
}
},
"systemMetadata": {
"lastObserved": 1718004101686,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
@ -582,17 +504,65 @@
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:b83da80a4d444484521d9f7aca958742",
"urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
},
{
"id": "Acryl Workbook"
}
]
}
},
"systemMetadata": {
"lastObserved": 1732545849249,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
"container": "urn:li:container:b83da80a4d444484521d9f7aca958742"
}
},
"systemMetadata": {
"lastObserved": 1718004101689,
"lastObserved": 1732513100096,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:b83da80a4d444484521d9f7aca958742",
"urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
},
{
"id": "Acryl Workbook"
}
]
}
},
"systemMetadata": {
"lastObserved": 1732545849248,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
@ -717,14 +687,13 @@
"urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
},
{
"id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
"id": "Acryl Workbook"
}
]
}
},
"systemMetadata": {
"lastObserved": 1718004101692,
"lastObserved": 1732545849252,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
@ -778,22 +747,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
}
},
"systemMetadata": {
"lastObserved": 1718004101695,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)",
@ -914,14 +867,13 @@
"urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
},
{
"id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
"id": "Acryl Workbook"
}
]
}
},
"systemMetadata": {
"lastObserved": 1718004101697,
"lastObserved": 1732545849255,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
@ -1155,6 +1107,7 @@
"urn:li:chart:(sigma,tQJu5N1l81)"
],
"datasets": [],
"dashboards": [],
"lastModified": {
"created": {
"time": 0,
@ -1174,17 +1127,17 @@
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)",
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
"changeType": "UPSERT",
"aspectName": "container",
"aspectName": "status",
"aspect": {
"json": {
"container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1718004101704,
"lastObserved": 1718004101706,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
@ -1202,30 +1155,13 @@
"urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
},
{
"id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
"id": "Acryl Workbook"
}
]
}
},
"systemMetadata": {
"lastObserved": 1718004101704,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1718004101706,
"lastObserved": 1732545849260,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
@ -1263,22 +1199,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
}
},
"systemMetadata": {
"lastObserved": 1718004101708,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
@ -1474,14 +1394,13 @@
"urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
},
{
"id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
"urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
"id": "Acryl Workbook"
}
]
}
},
"systemMetadata": {
"lastObserved": 1718004101712,
"lastObserved": 1732545849264,
"runId": "sigma-test",
"lastRunId": "no-run-id-provided"
}