diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index 8296bc8244..7fa1decdf7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -17,6 +17,7 @@ import com.linkedin.common.SubTypes; import com.linkedin.common.TimeStamp; import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; +import com.linkedin.datahub.graphql.generated.AuditStamp; import com.linkedin.datahub.graphql.generated.Container; import com.linkedin.datahub.graphql.generated.DataPlatform; import com.linkedin.datahub.graphql.generated.Dataset; @@ -200,10 +201,12 @@ public class DatasetMapper implements ModelMapper { } TimeStamp lastModified = gmsProperties.getLastModified(); if (lastModified != null) { - properties.setLastModified(lastModified.getTime()); - if (lastModified.hasActor()) { - properties.setLastModifiedActor(lastModified.getActor().toString()); - } + Urn actor = lastModified.getActor(); + properties.setLastModified( + new AuditStamp(lastModified.getTime(), actor == null ? null : actor.toString())); + properties.setLastModifiedActor(actor == null ? null : actor.toString()); + } else { + properties.setLastModified(new AuditStamp(0L, null)); } } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 4f3769d908..feb344154d 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -1789,12 +1789,13 @@ type DatasetProperties { """ Last Modified timestamp millis associated with the Dataset """ - lastModified: Long + lastModified: AuditStamp! """ - Actor associated with the Dataset's lastModified timestamp + Actor associated with the Dataset's lastModified timestamp. + Deprecated - Use lastModified.actor instead. """ - lastModifiedActor: String + lastModifiedActor: String @deprecated } @@ -11234,4 +11235,4 @@ input UpdateOwnershipTypeInput { The description of the Custom Ownership Type """ description: String -} \ No newline at end of file +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapperTest.java index 1959ae6d43..b28dd287e3 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapperTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapperTest.java @@ -2,6 +2,7 @@ package com.linkedin.datahub.graphql.types.dataset.mappers; import com.linkedin.common.TimeStamp; import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.AuditStamp; import com.linkedin.datahub.graphql.generated.Dataset; import com.linkedin.datahub.graphql.generated.DatasetProperties; import com.linkedin.entity.Aspect; @@ -58,7 +59,8 @@ public class DatasetMapperTest { expectedDatasetProperties.setQualifiedName("Test QualifiedName"); expectedDatasetProperties.setLastModifiedActor(TEST_LAST_MODIFIED_ACTOR_URN.toString()); expectedDatasetProperties.setCreatedActor(TEST_CREATED_ACTOR_URN.toString()); - expectedDatasetProperties.setLastModified(20L); + expectedDatasetProperties.setLastModified( + new AuditStamp(20L, TEST_LAST_MODIFIED_ACTOR_URN.toString())); expectedDatasetProperties.setCreated(10L); expected.setProperties(expectedDatasetProperties); @@ -68,7 +70,11 @@ public class DatasetMapperTest { actual.getProperties().getQualifiedName(), expected.getProperties().getQualifiedName()); Assert.assertEquals( - actual.getProperties().getLastModified(), expected.getProperties().getLastModified()); + actual.getProperties().getLastModified().getTime(), + expected.getProperties().getLastModified().getTime()); + Assert.assertEquals( + actual.getProperties().getLastModified().getActor(), + expected.getProperties().getLastModified().getActor()); Assert.assertEquals(actual.getProperties().getCreated(), expected.getProperties().getCreated()); Assert.assertEquals( @@ -102,7 +108,7 @@ public class DatasetMapperTest { expectedDatasetProperties.setName("Test"); expectedDatasetProperties.setLastModifiedActor(null); expectedDatasetProperties.setCreatedActor(null); - expectedDatasetProperties.setLastModified(null); + expectedDatasetProperties.setLastModified(new AuditStamp(0L, null)); expectedDatasetProperties.setCreated(null); expected.setProperties(expectedDatasetProperties); @@ -110,7 +116,11 @@ public class DatasetMapperTest { Assert.assertEquals(actual.getProperties().getName(), expected.getProperties().getName()); Assert.assertEquals( - actual.getProperties().getLastModified(), expected.getProperties().getLastModified()); + actual.getProperties().getLastModified().getTime(), + expected.getProperties().getLastModified().getTime()); + Assert.assertEquals( + actual.getProperties().getLastModified().getActor(), + expected.getProperties().getLastModified().getActor()); Assert.assertEquals(actual.getProperties().getCreated(), expected.getProperties().getCreated()); Assert.assertEquals( @@ -152,7 +162,7 @@ public class DatasetMapperTest { expectedDatasetProperties.setName("Test"); expectedDatasetProperties.setLastModifiedActor(null); expectedDatasetProperties.setCreatedActor(null); - expectedDatasetProperties.setLastModified(20L); + expectedDatasetProperties.setLastModified(new AuditStamp(20L, null)); expectedDatasetProperties.setCreated(10L); expected.setProperties(expectedDatasetProperties); @@ -160,7 +170,11 @@ public class DatasetMapperTest { Assert.assertEquals(actual.getProperties().getName(), expected.getProperties().getName()); Assert.assertEquals( - actual.getProperties().getLastModified(), expected.getProperties().getLastModified()); + actual.getProperties().getLastModified().getTime(), + expected.getProperties().getLastModified().getTime()); + Assert.assertEquals( + actual.getProperties().getLastModified().getActor(), + expected.getProperties().getLastModified().getActor()); Assert.assertEquals(actual.getProperties().getCreated(), expected.getProperties().getCreated()); Assert.assertEquals( diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json index c26338ea28..b949c9ab9d 100644 --- a/datahub-web-react/package.json +++ b/datahub-web-react/package.json @@ -92,6 +92,7 @@ "scripts": { "analyze": "source-map-explorer 'dist/static/js/*.js'", "start": "yarn run generate && BROWSER=none REACT_APP_MOCK=false craco start", + "start:dev": "yarn run generate && DISABLE_ESLINT_PLUGIN=true BROWSER=none REACT_APP_MOCK=false craco start", "start:mock": "yarn run generate && BROWSER=none REACT_APP_MOCK=true craco start", "start:e2e": "REACT_APP_MOCK=cy BROWSER=none PORT=3010 craco start", "ec2-dev": "yarn run generate && CI=true;export CI;BROWSER=none craco start", diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index ada9a06ab5..17173fd28e 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -437,6 +437,11 @@ export const dataset3 = { }, ], externalUrl: 'https://data.hub', + lastModified: { + __typename: 'AuditStamp', + time: 0, + actor: null, + }, }, parentContainers: { __typename: 'ParentContainersResult', @@ -702,6 +707,7 @@ export const dataset5 = { origin: 'PROD', customProperties: [{ key: 'propertyAKey', value: 'propertyAValue', associatedUrn: 'urn:li:dataset:5' }], externalUrl: 'https://data.hub', + lastModified: dataset3.properties?.lastModified, }, }; @@ -716,6 +722,7 @@ export const dataset6 = { origin: 'PROD', customProperties: [{ key: 'propertyAKey', value: 'propertyAValue', associatedUrn: 'urn:li:dataset:6' }], externalUrl: 'https://data.hub', + lastModified: dataset3.properties?.lastModified, }, }; diff --git a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx index 7d40b97a66..f60eb95937 100644 --- a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx +++ b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx @@ -33,6 +33,7 @@ import DataProductSection from '../shared/containers/profile/sidebar/DataProduct import { getDataProduct } from '../shared/utils'; import AccessManagement from '../shared/tabs/Dataset/AccessManagement/AccessManagement'; import { matchedFieldPathsRenderer } from '../../search/matches/matchedFieldPathsRenderer'; +import { getLastUpdatedMs } from './shared/utils'; const SUBTYPES = { VIEW: 'view', @@ -310,9 +311,7 @@ export class DatasetEntity implements Entity { rowCount={(data as any).lastProfile?.length && (data as any).lastProfile[0].rowCount} columnCount={(data as any).lastProfile?.length && (data as any).lastProfile[0].columnCount} sizeInBytes={(data as any).lastProfile?.length && (data as any).lastProfile[0].sizeInBytes} - lastUpdatedMs={ - (data as any).lastOperation?.length && (data as any).lastOperation[0].lastUpdatedTimestamp - } + lastUpdatedMs={getLastUpdatedMs(data.properties, (data as any)?.lastOperation)} health={data.health} degree={(result as any).degree} paths={(result as any).paths} diff --git a/datahub-web-react/src/app/entity/dataset/profile/stats/stats/DatasetStatsSummarySubHeader.tsx b/datahub-web-react/src/app/entity/dataset/profile/stats/stats/DatasetStatsSummarySubHeader.tsx index 36b7d25195..c1e2c1aa29 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/stats/stats/DatasetStatsSummarySubHeader.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/stats/stats/DatasetStatsSummarySubHeader.tsx @@ -3,6 +3,7 @@ import { DatasetStatsSummary as DatasetStatsSummaryObj } from '../../../../../.. import { useBaseEntity } from '../../../../shared/EntityContext'; import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated'; import { DatasetStatsSummary } from '../../../shared/DatasetStatsSummary'; +import { getLastUpdatedMs } from '../../../shared/utils'; export const DatasetStatsSummarySubHeader = () => { const result = useBaseEntity(); @@ -13,15 +14,13 @@ export const DatasetStatsSummarySubHeader = () => { const maybeLastProfile = dataset?.datasetProfiles && dataset.datasetProfiles.length ? dataset.datasetProfiles[0] : undefined; - const maybeLastOperation = dataset?.operations && dataset.operations.length ? dataset.operations[0] : undefined; - const rowCount = maybeLastProfile?.rowCount; const columnCount = maybeLastProfile?.columnCount; const sizeInBytes = maybeLastProfile?.sizeInBytes; const totalSqlQueries = dataset?.usageStats?.aggregations?.totalSqlQueries; const queryCountLast30Days = maybeStatsSummary?.queryCountLast30Days; const uniqueUserCountLast30Days = maybeStatsSummary?.uniqueUserCountLast30Days; - const lastUpdatedMs = maybeLastOperation?.lastUpdatedTimestamp; + const lastUpdatedMs = getLastUpdatedMs(dataset?.properties, dataset?.operations); return ( | null | undefined, + operations: Pick[] | null | undefined, +): number | undefined { + return ( + Math.max( + properties?.lastModified?.time || 0, + (operations?.length && operations[0].lastUpdatedTimestamp) || 0, + ) || undefined + ); +} diff --git a/datahub-web-react/src/app/search/autoComplete/AutoCompleteTooltipContent.tsx b/datahub-web-react/src/app/search/autoComplete/AutoCompleteTooltipContent.tsx index dfe32c7805..4e40c29722 100644 --- a/datahub-web-react/src/app/search/autoComplete/AutoCompleteTooltipContent.tsx +++ b/datahub-web-react/src/app/search/autoComplete/AutoCompleteTooltipContent.tsx @@ -3,6 +3,7 @@ import React from 'react'; import styled from 'styled-components'; import { Dataset, Entity, EntityType } from '../../../types.generated'; import { DatasetStatsSummary } from '../../entity/dataset/shared/DatasetStatsSummary'; +import { getLastUpdatedMs } from '../../entity/dataset/shared/utils'; import { useEntityRegistry } from '../../useEntityRegistry'; import { ArrowWrapper } from './ParentContainers'; @@ -48,9 +49,7 @@ export default function AutoCompleteTooltipContent({ entity }: Props) { rowCount={(entity as any).lastProfile?.length && (entity as any).lastProfile[0].rowCount} columnCount={(entity as any).lastProfile?.length && (entity as any).lastProfile[0].columnCount} sizeInBytes={(entity as any).lastProfile?.length && (entity as any).lastProfile[0].sizeInBytes} - lastUpdatedMs={ - (entity as any).lastOperation?.length && (entity as any).lastOperation[0].lastUpdatedTimestamp - } + lastUpdatedMs={getLastUpdatedMs((entity as any)?.properties, (entity as any)?.lastOperation)} queryCountLast30Days={(entity as Dataset).statsSummary?.queryCountLast30Days} uniqueUserCountLast30Days={(entity as Dataset).statsSummary?.uniqueUserCountLast30Days} mode="tooltip-content" diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index d693779d11..b77ef9d1ad 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -240,6 +240,10 @@ fragment nonRecursiveDatasetFields on Dataset { value } externalUrl + lastModified { + time + actor + } } editableProperties { description diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index 6ca2a78f93..7034116f76 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -13,6 +13,10 @@ fragment autoCompleteFields on Entity { properties { name qualifiedName + lastModified { + time + actor + } } parentContainers { ...parentContainersFields @@ -39,6 +43,10 @@ fragment autoCompleteFields on Entity { description qualifiedName externalUrl + lastModified { + time + actor + } } } } @@ -336,6 +344,10 @@ fragment nonSiblingsDatasetSearchFields on Dataset { value } externalUrl + lastModified { + time + actor + } } ownership { ...ownershipFields diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py index 03b4f61a51..d1940c1d57 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py @@ -1,6 +1,5 @@ import logging import re -import time from concurrent.futures import ThreadPoolExecutor from typing import Dict, Iterable, List, Optional, Set, Union from urllib.parse import urljoin @@ -87,8 +86,6 @@ from datahub.metadata.schema_classes import ( DomainsClass, MySqlDDLClass, NullTypeClass, - OperationClass, - OperationTypeClass, OwnerClass, OwnershipClass, OwnershipTypeClass, @@ -402,7 +399,6 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource): sub_type = self._create_table_sub_type_aspect(table) schema_metadata = self._create_schema_metadata_aspect(table) - operation = self._create_table_operation_aspect(table) domain = self._get_domain_aspect(dataset_name=table.ref.qualified_table_name) ownership = self._create_table_ownership_aspect(table) data_platform_instance = self._create_data_platform_instance_aspect() @@ -424,7 +420,6 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource): view_props, sub_type, schema_metadata, - operation, domain, ownership, data_platform_instance, @@ -696,10 +691,10 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource): int(table.created_at.timestamp() * 1000), make_user_urn(table.created_by) ) last_modified = created - if table.updated_at and table.updated_by is not None: + if table.updated_at: last_modified = TimeStampClass( int(table.updated_at.timestamp() * 1000), - make_user_urn(table.updated_by), + table.updated_by and make_user_urn(table.updated_by), ) return DatasetPropertiesClass( @@ -712,35 +707,6 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource): externalUrl=f"{self.external_url_base}/{table.ref.external_path}", ) - def _create_table_operation_aspect(self, table: Table) -> OperationClass: - """Produce an operation aspect for a table. - - If a last updated time is present, we produce an update operation. - Otherwise, we produce a create operation. We do this in addition to - setting the last updated time in the dataset properties aspect, as - the UI is currently missing the ability to display the last updated - from the properties aspect. - """ - - reported_time = int(time.time() * 1000) - - operation = OperationClass( - timestampMillis=reported_time, - lastUpdatedTimestamp=int(table.created_at.timestamp() * 1000), - actor=make_user_urn(table.created_by), - operationType=OperationTypeClass.CREATE, - ) - - if table.updated_at and table.updated_by is not None: - operation = OperationClass( - timestampMillis=reported_time, - lastUpdatedTimestamp=int(table.updated_at.timestamp() * 1000), - actor=make_user_urn(table.updated_by), - operationType=OperationTypeClass.UPDATE, - ) - - return operation - def _create_table_ownership_aspect(self, table: Table) -> Optional[OwnershipClass]: owner_urn = self.get_owner_urn(table.owner) if owner_urn is not None: diff --git a/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json b/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json index 2e92215d70..d25c86a3a1 100644 --- a/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json +++ b/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json @@ -524,29 +524,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.default.quickstart_table,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1638860400000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@acryl.io", - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1666186049633 - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.default.quickstart_table,PROD)", @@ -877,29 +854,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.information_schema.quickstart_table,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1638860400000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@acryl.io", - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1666186049633 - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.information_schema.quickstart_table,PROD)", @@ -1230,29 +1184,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.quickstart_schema.quickstart_table,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1638860400000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@acryl.io", - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1666186049633 - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.quickstart_schema.quickstart_table,PROD)", @@ -1719,29 +1650,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.default.quickstart_table,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1638860400000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@acryl.io", - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1666186049633 - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.default.quickstart_table,PROD)", @@ -2072,29 +1980,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.information_schema.quickstart_table,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1638860400000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@acryl.io", - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1666186049633 - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.information_schema.quickstart_table,PROD)", @@ -2425,29 +2310,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.quickstart_schema.quickstart_table,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1638860400000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@acryl.io", - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1666186049633 - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.quickstart_schema.quickstart_table,PROD)", @@ -2914,29 +2776,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.default.quickstart_table,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1638860400000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@acryl.io", - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1666186049633 - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.default.quickstart_table,PROD)", @@ -3267,29 +3106,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.information_schema.quickstart_table,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1638860400000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@acryl.io", - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1666186049633 - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.information_schema.quickstart_table,PROD)", @@ -3620,29 +3436,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.quickstart_schema.quickstart_table,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1638860400000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@acryl.io", - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1666186049633 - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.quickstart_schema.quickstart_table,PROD)", diff --git a/metadata-ingestion/tests/unit/serde/test_serde.py b/metadata-ingestion/tests/unit/serde/test_serde.py index d2d6a0bdda..53ffdf46a6 100644 --- a/metadata-ingestion/tests/unit/serde/test_serde.py +++ b/metadata-ingestion/tests/unit/serde/test_serde.py @@ -100,7 +100,7 @@ def test_serde_to_avro( fo.seek(0) in_records = list(fastavro.reader(fo, return_record_name=True)) in_mces = [ - MetadataChangeEventClass.from_obj(record, tuples=True) + MetadataChangeEventClass.from_obj(record, tuples=True) # type: ignore for record in in_records ] diff --git a/metadata-service/openapi-entity-servlet/build.gradle b/metadata-service/openapi-entity-servlet/build.gradle index dbec469085..00353392de 100644 --- a/metadata-service/openapi-entity-servlet/build.gradle +++ b/metadata-service/openapi-entity-servlet/build.gradle @@ -77,4 +77,4 @@ task openApiGenerate(type: GenerateSwaggerCode, dependsOn: [mergeApiComponents, 'delegatePattern' : "false" ] } -tasks.getByName("compileJava").dependsOn(openApiGenerate) \ No newline at end of file +tasks.getByName("compileJava").dependsOn(openApiGenerate)