feat(ui/last-updated): Calculate last updated time as max(properties time, operation time) (#9242)

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java

@@ -17,6 +17,7 @@ import com.linkedin.common.SubTypes;
 import com.linkedin.common.TimeStamp;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.DataMap;
+import com.linkedin.datahub.graphql.generated.AuditStamp;
 import com.linkedin.datahub.graphql.generated.Container;
 import com.linkedin.datahub.graphql.generated.DataPlatform;
 import com.linkedin.datahub.graphql.generated.Dataset;
@@ -200,10 +201,12 @@ public class DatasetMapper implements ModelMapper<EntityResponse, Dataset> {
     }
     TimeStamp lastModified = gmsProperties.getLastModified();
     if (lastModified != null) {
-      properties.setLastModified(lastModified.getTime());
+      Urn actor = lastModified.getActor();
-      if (lastModified.hasActor()) {
+      properties.setLastModified(
-        properties.setLastModifiedActor(lastModified.getActor().toString());
+          new AuditStamp(lastModified.getTime(), actor == null ? null : actor.toString()));
-      }
+      properties.setLastModifiedActor(actor == null ? null : actor.toString());
+    } else {
+      properties.setLastModified(new AuditStamp(0L, null));
     }
   }
 

datahub-graphql-core/src/main/resources/entity.graphql

@@ -1789,12 +1789,13 @@ type DatasetProperties {
     """
     Last Modified timestamp millis associated with the Dataset
     """
-    lastModified: Long
+    lastModified: AuditStamp!
 
     """
-    Actor associated with the Dataset's lastModified timestamp
+    Actor associated with the Dataset's lastModified timestamp.
+    Deprecated - Use lastModified.actor instead.
     """
-    lastModifiedActor: String
+    lastModifiedActor: String @deprecated
 }
 
 

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapperTest.java

@@ -2,6 +2,7 @@ package com.linkedin.datahub.graphql.types.dataset.mappers;
 
 import com.linkedin.common.TimeStamp;
 import com.linkedin.common.urn.Urn;
+import com.linkedin.datahub.graphql.generated.AuditStamp;
 import com.linkedin.datahub.graphql.generated.Dataset;
 import com.linkedin.datahub.graphql.generated.DatasetProperties;
 import com.linkedin.entity.Aspect;
@@ -58,7 +59,8 @@ public class DatasetMapperTest {
     expectedDatasetProperties.setQualifiedName("Test QualifiedName");
     expectedDatasetProperties.setLastModifiedActor(TEST_LAST_MODIFIED_ACTOR_URN.toString());
     expectedDatasetProperties.setCreatedActor(TEST_CREATED_ACTOR_URN.toString());
-    expectedDatasetProperties.setLastModified(20L);
+    expectedDatasetProperties.setLastModified(
+        new AuditStamp(20L, TEST_LAST_MODIFIED_ACTOR_URN.toString()));
     expectedDatasetProperties.setCreated(10L);
     expected.setProperties(expectedDatasetProperties);
 
@@ -68,7 +70,11 @@ public class DatasetMapperTest {
         actual.getProperties().getQualifiedName(), expected.getProperties().getQualifiedName());
 
     Assert.assertEquals(
-        actual.getProperties().getLastModified(), expected.getProperties().getLastModified());
+        actual.getProperties().getLastModified().getTime(),
+        expected.getProperties().getLastModified().getTime());
+    Assert.assertEquals(
+        actual.getProperties().getLastModified().getActor(),
+        expected.getProperties().getLastModified().getActor());
     Assert.assertEquals(actual.getProperties().getCreated(), expected.getProperties().getCreated());
 
     Assert.assertEquals(
@@ -102,7 +108,7 @@ public class DatasetMapperTest {
     expectedDatasetProperties.setName("Test");
     expectedDatasetProperties.setLastModifiedActor(null);
     expectedDatasetProperties.setCreatedActor(null);
-    expectedDatasetProperties.setLastModified(null);
+    expectedDatasetProperties.setLastModified(new AuditStamp(0L, null));
     expectedDatasetProperties.setCreated(null);
     expected.setProperties(expectedDatasetProperties);
 
@@ -110,7 +116,11 @@ public class DatasetMapperTest {
     Assert.assertEquals(actual.getProperties().getName(), expected.getProperties().getName());
 
     Assert.assertEquals(
-        actual.getProperties().getLastModified(), expected.getProperties().getLastModified());
+        actual.getProperties().getLastModified().getTime(),
+        expected.getProperties().getLastModified().getTime());
+    Assert.assertEquals(
+        actual.getProperties().getLastModified().getActor(),
+        expected.getProperties().getLastModified().getActor());
     Assert.assertEquals(actual.getProperties().getCreated(), expected.getProperties().getCreated());
 
     Assert.assertEquals(
@@ -152,7 +162,7 @@ public class DatasetMapperTest {
     expectedDatasetProperties.setName("Test");
     expectedDatasetProperties.setLastModifiedActor(null);
     expectedDatasetProperties.setCreatedActor(null);
-    expectedDatasetProperties.setLastModified(20L);
+    expectedDatasetProperties.setLastModified(new AuditStamp(20L, null));
     expectedDatasetProperties.setCreated(10L);
     expected.setProperties(expectedDatasetProperties);
 
@@ -160,7 +170,11 @@ public class DatasetMapperTest {
     Assert.assertEquals(actual.getProperties().getName(), expected.getProperties().getName());
 
     Assert.assertEquals(
-        actual.getProperties().getLastModified(), expected.getProperties().getLastModified());
+        actual.getProperties().getLastModified().getTime(),
+        expected.getProperties().getLastModified().getTime());
+    Assert.assertEquals(
+        actual.getProperties().getLastModified().getActor(),
+        expected.getProperties().getLastModified().getActor());
     Assert.assertEquals(actual.getProperties().getCreated(), expected.getProperties().getCreated());
 
     Assert.assertEquals(

datahub-web-react/package.json

@@ -92,6 +92,7 @@
     "scripts": {
         "analyze": "source-map-explorer 'dist/static/js/*.js'",
         "start": "yarn run generate && BROWSER=none REACT_APP_MOCK=false craco start",
+        "start:dev": "yarn run generate && DISABLE_ESLINT_PLUGIN=true BROWSER=none REACT_APP_MOCK=false craco start",
         "start:mock": "yarn run generate && BROWSER=none REACT_APP_MOCK=true craco start",
         "start:e2e": "REACT_APP_MOCK=cy BROWSER=none PORT=3010 craco start",
         "ec2-dev": "yarn run generate && CI=true;export CI;BROWSER=none craco start",

datahub-web-react/src/Mocks.tsx

@@ -437,6 +437,11 @@ export const dataset3 = {
             },
         ],
         externalUrl: 'https://data.hub',
+        lastModified: {
+            __typename: 'AuditStamp',
+            time: 0,
+            actor: null,
+        },
     },
     parentContainers: {
         __typename: 'ParentContainersResult',
@@ -702,6 +707,7 @@ export const dataset5 = {
         origin: 'PROD',
         customProperties: [{ key: 'propertyAKey', value: 'propertyAValue', associatedUrn: 'urn:li:dataset:5' }],
         externalUrl: 'https://data.hub',
+        lastModified: dataset3.properties?.lastModified,
     },
 };
 
@@ -716,6 +722,7 @@ export const dataset6 = {
         origin: 'PROD',
         customProperties: [{ key: 'propertyAKey', value: 'propertyAValue', associatedUrn: 'urn:li:dataset:6' }],
         externalUrl: 'https://data.hub',
+        lastModified: dataset3.properties?.lastModified,
     },
 };
 

datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx

@@ -33,6 +33,7 @@ import DataProductSection from '../shared/containers/profile/sidebar/DataProduct
 import { getDataProduct } from '../shared/utils';
 import AccessManagement from '../shared/tabs/Dataset/AccessManagement/AccessManagement';
 import { matchedFieldPathsRenderer } from '../../search/matches/matchedFieldPathsRenderer';
+import { getLastUpdatedMs } from './shared/utils';
 
 const SUBTYPES = {
     VIEW: 'view',
@@ -310,9 +311,7 @@ export class DatasetEntity implements Entity<Dataset> {
                 rowCount={(data as any).lastProfile?.length && (data as any).lastProfile[0].rowCount}
                 columnCount={(data as any).lastProfile?.length && (data as any).lastProfile[0].columnCount}
                 sizeInBytes={(data as any).lastProfile?.length && (data as any).lastProfile[0].sizeInBytes}
-                lastUpdatedMs={
+                lastUpdatedMs={getLastUpdatedMs(data.properties, (data as any)?.lastOperation)}
-                    (data as any).lastOperation?.length && (data as any).lastOperation[0].lastUpdatedTimestamp
-                }
                 health={data.health}
                 degree={(result as any).degree}
                 paths={(result as any).paths}

datahub-web-react/src/app/entity/dataset/profile/stats/stats/DatasetStatsSummarySubHeader.tsx

@@ -3,6 +3,7 @@ import { DatasetStatsSummary as DatasetStatsSummaryObj } from '../../../../../..
 import { useBaseEntity } from '../../../../shared/EntityContext';
 import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated';
 import { DatasetStatsSummary } from '../../../shared/DatasetStatsSummary';
+import { getLastUpdatedMs } from '../../../shared/utils';
 
 export const DatasetStatsSummarySubHeader = () => {
     const result = useBaseEntity<GetDatasetQuery>();
@@ -13,15 +14,13 @@ export const DatasetStatsSummarySubHeader = () => {
     const maybeLastProfile =
         dataset?.datasetProfiles && dataset.datasetProfiles.length ? dataset.datasetProfiles[0] : undefined;
 
-    const maybeLastOperation = dataset?.operations && dataset.operations.length ? dataset.operations[0] : undefined;
-
     const rowCount = maybeLastProfile?.rowCount;
     const columnCount = maybeLastProfile?.columnCount;
     const sizeInBytes = maybeLastProfile?.sizeInBytes;
     const totalSqlQueries = dataset?.usageStats?.aggregations?.totalSqlQueries;
     const queryCountLast30Days = maybeStatsSummary?.queryCountLast30Days;
     const uniqueUserCountLast30Days = maybeStatsSummary?.uniqueUserCountLast30Days;
-    const lastUpdatedMs = maybeLastOperation?.lastUpdatedTimestamp;
+    const lastUpdatedMs = getLastUpdatedMs(dataset?.properties, dataset?.operations);
 
     return (
         <DatasetStatsSummary

datahub-web-react/src/app/entity/dataset/shared/utils.ts

@@ -0,0 +1,13 @@
+import { DatasetProperties, Operation } from '../../../../types.generated';
+
+export function getLastUpdatedMs(
+    properties: Pick<DatasetProperties, 'lastModified'> | null | undefined,
+    operations: Pick<Operation, 'lastUpdatedTimestamp'>[] | null | undefined,
+): number | undefined {
+    return (
+        Math.max(
+            properties?.lastModified?.time || 0,
+            (operations?.length && operations[0].lastUpdatedTimestamp) || 0,
+        ) || undefined
+    );
+}

datahub-web-react/src/app/search/autoComplete/AutoCompleteTooltipContent.tsx

@@ -3,6 +3,7 @@ import React from 'react';
 import styled from 'styled-components';
 import { Dataset, Entity, EntityType } from '../../../types.generated';
 import { DatasetStatsSummary } from '../../entity/dataset/shared/DatasetStatsSummary';
+import { getLastUpdatedMs } from '../../entity/dataset/shared/utils';
 import { useEntityRegistry } from '../../useEntityRegistry';
 import { ArrowWrapper } from './ParentContainers';
 
@@ -48,9 +49,7 @@ export default function AutoCompleteTooltipContent({ entity }: Props) {
                     rowCount={(entity as any).lastProfile?.length && (entity as any).lastProfile[0].rowCount}
                     columnCount={(entity as any).lastProfile?.length && (entity as any).lastProfile[0].columnCount}
                     sizeInBytes={(entity as any).lastProfile?.length && (entity as any).lastProfile[0].sizeInBytes}
-                    lastUpdatedMs={
+                    lastUpdatedMs={getLastUpdatedMs((entity as any)?.properties, (entity as any)?.lastOperation)}
-                        (entity as any).lastOperation?.length && (entity as any).lastOperation[0].lastUpdatedTimestamp
-                    }
                     queryCountLast30Days={(entity as Dataset).statsSummary?.queryCountLast30Days}
                     uniqueUserCountLast30Days={(entity as Dataset).statsSummary?.uniqueUserCountLast30Days}
                     mode="tooltip-content"

datahub-web-react/src/graphql/fragments.graphql

@@ -240,6 +240,10 @@ fragment nonRecursiveDatasetFields on Dataset {
             value
         }
         externalUrl
+        lastModified {
+            time
+            actor
+        }
     }
     editableProperties {
         description

datahub-web-react/src/graphql/search.graphql

@@ -13,6 +13,10 @@ fragment autoCompleteFields on Entity {
         properties {
             name
             qualifiedName
+            lastModified {
+                time
+                actor
+            }
         }
         parentContainers {
             ...parentContainersFields
@@ -39,6 +43,10 @@ fragment autoCompleteFields on Entity {
                         description
                         qualifiedName
                         externalUrl
+                        lastModified {
+                            time
+                            actor
+                        }
                     }
                 }
             }
@@ -336,6 +344,10 @@ fragment nonSiblingsDatasetSearchFields on Dataset {
             value
         }
         externalUrl
+        lastModified {
+            time
+            actor
+        }
     }
     ownership {
         ...ownershipFields

metadata-ingestion/src/datahub/ingestion/source/unity/source.py

@@ -1,6 +1,5 @@
 import logging
 import re
-import time
 from concurrent.futures import ThreadPoolExecutor
 from typing import Dict, Iterable, List, Optional, Set, Union
 from urllib.parse import urljoin
@@ -87,8 +86,6 @@ from datahub.metadata.schema_classes import (
     DomainsClass,
     MySqlDDLClass,
     NullTypeClass,
-    OperationClass,
-    OperationTypeClass,
     OwnerClass,
     OwnershipClass,
     OwnershipTypeClass,
@@ -402,7 +399,6 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
 
         sub_type = self._create_table_sub_type_aspect(table)
         schema_metadata = self._create_schema_metadata_aspect(table)
-        operation = self._create_table_operation_aspect(table)
         domain = self._get_domain_aspect(dataset_name=table.ref.qualified_table_name)
         ownership = self._create_table_ownership_aspect(table)
         data_platform_instance = self._create_data_platform_instance_aspect()
@@ -424,7 +420,6 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
                     view_props,
                     sub_type,
                     schema_metadata,
-                    operation,
                     domain,
                     ownership,
                     data_platform_instance,
@@ -696,10 +691,10 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
             int(table.created_at.timestamp() * 1000), make_user_urn(table.created_by)
         )
         last_modified = created
-        if table.updated_at and table.updated_by is not None:
+        if table.updated_at:
             last_modified = TimeStampClass(
                 int(table.updated_at.timestamp() * 1000),
-                make_user_urn(table.updated_by),
+                table.updated_by and make_user_urn(table.updated_by),
             )
 
         return DatasetPropertiesClass(
@@ -712,35 +707,6 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
             externalUrl=f"{self.external_url_base}/{table.ref.external_path}",
         )
 
-    def _create_table_operation_aspect(self, table: Table) -> OperationClass:
-        """Produce an operation aspect for a table.
-
-        If a last updated time is present, we produce an update operation.
-        Otherwise, we produce a create operation. We do this in addition to
-        setting the last updated time in the dataset properties aspect, as
-        the UI is currently missing the ability to display the last updated
-        from the properties aspect.
-        """
-
-        reported_time = int(time.time() * 1000)
-
-        operation = OperationClass(
-            timestampMillis=reported_time,
-            lastUpdatedTimestamp=int(table.created_at.timestamp() * 1000),
-            actor=make_user_urn(table.created_by),
-            operationType=OperationTypeClass.CREATE,
-        )
-
-        if table.updated_at and table.updated_by is not None:
-            operation = OperationClass(
-                timestampMillis=reported_time,
-                lastUpdatedTimestamp=int(table.updated_at.timestamp() * 1000),
-                actor=make_user_urn(table.updated_by),
-                operationType=OperationTypeClass.UPDATE,
-            )
-
-        return operation
-
     def _create_table_ownership_aspect(self, table: Table) -> Optional[OwnershipClass]:
         owner_urn = self.get_owner_urn(table.owner)
         if owner_urn is not None:

metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json

@@ -524,29 +524,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.default.quickstart_table,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1638860400000,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "actor": "urn:li:corpuser:abc@acryl.io",
-            "operationType": "UPDATE",
-            "lastUpdatedTimestamp": 1666186049633
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.default.quickstart_table,PROD)",
@@ -877,29 +854,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.information_schema.quickstart_table,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1638860400000,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "actor": "urn:li:corpuser:abc@acryl.io",
-            "operationType": "UPDATE",
-            "lastUpdatedTimestamp": 1666186049633
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.information_schema.quickstart_table,PROD)",
@@ -1230,29 +1184,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.quickstart_schema.quickstart_table,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1638860400000,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "actor": "urn:li:corpuser:abc@acryl.io",
-            "operationType": "UPDATE",
-            "lastUpdatedTimestamp": 1666186049633
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.quickstart_schema.quickstart_table,PROD)",
@@ -1719,29 +1650,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.default.quickstart_table,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1638860400000,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "actor": "urn:li:corpuser:abc@acryl.io",
-            "operationType": "UPDATE",
-            "lastUpdatedTimestamp": 1666186049633
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.default.quickstart_table,PROD)",
@@ -2072,29 +1980,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.information_schema.quickstart_table,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1638860400000,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "actor": "urn:li:corpuser:abc@acryl.io",
-            "operationType": "UPDATE",
-            "lastUpdatedTimestamp": 1666186049633
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.information_schema.quickstart_table,PROD)",
@@ -2425,29 +2310,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.quickstart_schema.quickstart_table,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1638860400000,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "actor": "urn:li:corpuser:abc@acryl.io",
-            "operationType": "UPDATE",
-            "lastUpdatedTimestamp": 1666186049633
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.quickstart_schema.quickstart_table,PROD)",
@@ -2914,29 +2776,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.default.quickstart_table,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1638860400000,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "actor": "urn:li:corpuser:abc@acryl.io",
-            "operationType": "UPDATE",
-            "lastUpdatedTimestamp": 1666186049633
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.default.quickstart_table,PROD)",
@@ -3267,29 +3106,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.information_schema.quickstart_table,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1638860400000,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "actor": "urn:li:corpuser:abc@acryl.io",
-            "operationType": "UPDATE",
-            "lastUpdatedTimestamp": 1666186049633
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.information_schema.quickstart_table,PROD)",
@@ -3620,29 +3436,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.quickstart_schema.quickstart_table,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1638860400000,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "actor": "urn:li:corpuser:abc@acryl.io",
-            "operationType": "UPDATE",
-            "lastUpdatedTimestamp": 1666186049633
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.quickstart_schema.quickstart_table,PROD)",

metadata-ingestion/tests/unit/serde/test_serde.py

@@ -100,7 +100,7 @@ def test_serde_to_avro(
         fo.seek(0)
         in_records = list(fastavro.reader(fo, return_record_name=True))
         in_mces = [
-            MetadataChangeEventClass.from_obj(record, tuples=True)
+            MetadataChangeEventClass.from_obj(record, tuples=True)  # type: ignore
             for record in in_records
         ]