yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-02 11:49:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ui/last-updated): Calculate last updated time as max(properties time, operation time) (#9242)

Browse Source
This commit is contained in:
Andrew Sikowitz 2023-12-06 15:21:56 -05:00 committed by GitHub
parent 27f23ecdd5
commit 16fe22aafa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 79 additions and 268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java
View File

@ -17,6 +17,7 @@ import com.linkedin.common.SubTypes;
import com.linkedin.common.TimeStamp;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.DataMap;
import com.linkedin.datahub.graphql.generated.AuditStamp;
import com.linkedin.datahub.graphql.generated.Container;
import com.linkedin.datahub.graphql.generated.DataPlatform;
import com.linkedin.datahub.graphql.generated.Dataset;
@ -200,10 +201,12 @@ public class DatasetMapper implements ModelMapper<EntityResponse, Dataset> {
}
TimeStamp lastModified = gmsProperties.getLastModified();
if (lastModified != null) {
properties.setLastModified(lastModified.getTime());
if (lastModified.hasActor()) {
properties.setLastModifiedActor(lastModified.getActor().toString());
}
Urn actor = lastModified.getActor();
properties.setLastModified(
new AuditStamp(lastModified.getTime(), actor == null ? null : actor.toString()));
properties.setLastModifiedActor(actor == null ? null : actor.toString());
} else {
properties.setLastModified(new AuditStamp(0L, null));
}
}

9
datahub-graphql-core/src/main/resources/entity.graphql
View File

@ -1789,12 +1789,13 @@ type DatasetProperties {
"""
Last Modified timestamp millis associated with the Dataset
"""
lastModified: Long
lastModified: AuditStamp!
"""
Actor associated with the Dataset's lastModified timestamp
Actor associated with the Dataset's lastModified timestamp.
Deprecated - Use lastModified.actor instead.
"""
lastModifiedActor: String
lastModifiedActor: String @deprecated
}
@ -11234,4 +11235,4 @@ input UpdateOwnershipTypeInput {
The description of the Custom Ownership Type
"""
description: String
}
}

26
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapperTest.java
View File

@ -2,6 +2,7 @@ package com.linkedin.datahub.graphql.types.dataset.mappers;
import com.linkedin.common.TimeStamp;
import com.linkedin.common.urn.Urn;
import com.linkedin.datahub.graphql.generated.AuditStamp;
import com.linkedin.datahub.graphql.generated.Dataset;
import com.linkedin.datahub.graphql.generated.DatasetProperties;
import com.linkedin.entity.Aspect;
@ -58,7 +59,8 @@ public class DatasetMapperTest {
expectedDatasetProperties.setQualifiedName("Test QualifiedName");
expectedDatasetProperties.setLastModifiedActor(TEST_LAST_MODIFIED_ACTOR_URN.toString());
expectedDatasetProperties.setCreatedActor(TEST_CREATED_ACTOR_URN.toString());
expectedDatasetProperties.setLastModified(20L);
expectedDatasetProperties.setLastModified(
new AuditStamp(20L, TEST_LAST_MODIFIED_ACTOR_URN.toString()));
expectedDatasetProperties.setCreated(10L);
expected.setProperties(expectedDatasetProperties);
@ -68,7 +70,11 @@ public class DatasetMapperTest {
actual.getProperties().getQualifiedName(), expected.getProperties().getQualifiedName());
Assert.assertEquals(
actual.getProperties().getLastModified(), expected.getProperties().getLastModified());
actual.getProperties().getLastModified().getTime(),
expected.getProperties().getLastModified().getTime());
Assert.assertEquals(
actual.getProperties().getLastModified().getActor(),
expected.getProperties().getLastModified().getActor());
Assert.assertEquals(actual.getProperties().getCreated(), expected.getProperties().getCreated());
Assert.assertEquals(
@ -102,7 +108,7 @@ public class DatasetMapperTest {
expectedDatasetProperties.setName("Test");
expectedDatasetProperties.setLastModifiedActor(null);
expectedDatasetProperties.setCreatedActor(null);
expectedDatasetProperties.setLastModified(null);
expectedDatasetProperties.setLastModified(new AuditStamp(0L, null));
expectedDatasetProperties.setCreated(null);
expected.setProperties(expectedDatasetProperties);
@ -110,7 +116,11 @@ public class DatasetMapperTest {
Assert.assertEquals(actual.getProperties().getName(), expected.getProperties().getName());
Assert.assertEquals(
actual.getProperties().getLastModified(), expected.getProperties().getLastModified());
actual.getProperties().getLastModified().getTime(),
expected.getProperties().getLastModified().getTime());
Assert.assertEquals(
actual.getProperties().getLastModified().getActor(),
expected.getProperties().getLastModified().getActor());
Assert.assertEquals(actual.getProperties().getCreated(), expected.getProperties().getCreated());
Assert.assertEquals(
@ -152,7 +162,7 @@ public class DatasetMapperTest {
expectedDatasetProperties.setName("Test");
expectedDatasetProperties.setLastModifiedActor(null);
expectedDatasetProperties.setCreatedActor(null);
expectedDatasetProperties.setLastModified(20L);
expectedDatasetProperties.setLastModified(new AuditStamp(20L, null));
expectedDatasetProperties.setCreated(10L);
expected.setProperties(expectedDatasetProperties);
@ -160,7 +170,11 @@ public class DatasetMapperTest {
Assert.assertEquals(actual.getProperties().getName(), expected.getProperties().getName());
Assert.assertEquals(
actual.getProperties().getLastModified(), expected.getProperties().getLastModified());
actual.getProperties().getLastModified().getTime(),
expected.getProperties().getLastModified().getTime());
Assert.assertEquals(
actual.getProperties().getLastModified().getActor(),
expected.getProperties().getLastModified().getActor());
Assert.assertEquals(actual.getProperties().getCreated(), expected.getProperties().getCreated());
Assert.assertEquals(

1
datahub-web-react/package.json
View File

@ -92,6 +92,7 @@
"scripts": {
"analyze": "source-map-explorer 'dist/static/js/*.js'",
"start": "yarn run generate && BROWSER=none REACT_APP_MOCK=false craco start",
"start:dev": "yarn run generate && DISABLE_ESLINT_PLUGIN=true BROWSER=none REACT_APP_MOCK=false craco start",
"start:mock": "yarn run generate && BROWSER=none REACT_APP_MOCK=true craco start",
"start:e2e": "REACT_APP_MOCK=cy BROWSER=none PORT=3010 craco start",
"ec2-dev": "yarn run generate && CI=true;export CI;BROWSER=none craco start",

7
datahub-web-react/src/Mocks.tsx
View File

@ -437,6 +437,11 @@ export const dataset3 = {
},
],
externalUrl: 'https://data.hub',
lastModified: {
__typename: 'AuditStamp',
time: 0,
actor: null,
},
},
parentContainers: {
__typename: 'ParentContainersResult',
@ -702,6 +707,7 @@ export const dataset5 = {
origin: 'PROD',
customProperties: [{ key: 'propertyAKey', value: 'propertyAValue', associatedUrn: 'urn:li:dataset:5' }],
externalUrl: 'https://data.hub',
lastModified: dataset3.properties?.lastModified,
},
};
@ -716,6 +722,7 @@ export const dataset6 = {
origin: 'PROD',
customProperties: [{ key: 'propertyAKey', value: 'propertyAValue', associatedUrn: 'urn:li:dataset:6' }],
externalUrl: 'https://data.hub',
lastModified: dataset3.properties?.lastModified,
},
};

5
datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx
View File

@ -33,6 +33,7 @@ import DataProductSection from '../shared/containers/profile/sidebar/DataProduct
import { getDataProduct } from '../shared/utils';
import AccessManagement from '../shared/tabs/Dataset/AccessManagement/AccessManagement';
import { matchedFieldPathsRenderer } from '../../search/matches/matchedFieldPathsRenderer';
import { getLastUpdatedMs } from './shared/utils';
const SUBTYPES = {
VIEW: 'view',
@ -310,9 +311,7 @@ export class DatasetEntity implements Entity<Dataset> {
rowCount={(data as any).lastProfile?.length && (data as any).lastProfile[0].rowCount}
columnCount={(data as any).lastProfile?.length && (data as any).lastProfile[0].columnCount}
sizeInBytes={(data as any).lastProfile?.length && (data as any).lastProfile[0].sizeInBytes}
lastUpdatedMs={
(data as any).lastOperation?.length && (data as any).lastOperation[0].lastUpdatedTimestamp
}
lastUpdatedMs={getLastUpdatedMs(data.properties, (data as any)?.lastOperation)}
health={data.health}
degree={(result as any).degree}
paths={(result as any).paths}

5
datahub-web-react/src/app/entity/dataset/profile/stats/stats/DatasetStatsSummarySubHeader.tsx
View File

@ -3,6 +3,7 @@ import { DatasetStatsSummary as DatasetStatsSummaryObj } from '../../../../../..
import { useBaseEntity } from '../../../../shared/EntityContext';
import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated';
import { DatasetStatsSummary } from '../../../shared/DatasetStatsSummary';
import { getLastUpdatedMs } from '../../../shared/utils';
export const DatasetStatsSummarySubHeader = () => {
const result = useBaseEntity<GetDatasetQuery>();
@ -13,15 +14,13 @@ export const DatasetStatsSummarySubHeader = () => {
const maybeLastProfile =
dataset?.datasetProfiles && dataset.datasetProfiles.length ? dataset.datasetProfiles[0] : undefined;
const maybeLastOperation = dataset?.operations && dataset.operations.length ? dataset.operations[0] : undefined;
const rowCount = maybeLastProfile?.rowCount;
const columnCount = maybeLastProfile?.columnCount;
const sizeInBytes = maybeLastProfile?.sizeInBytes;
const totalSqlQueries = dataset?.usageStats?.aggregations?.totalSqlQueries;
const queryCountLast30Days = maybeStatsSummary?.queryCountLast30Days;
const uniqueUserCountLast30Days = maybeStatsSummary?.uniqueUserCountLast30Days;
const lastUpdatedMs = maybeLastOperation?.lastUpdatedTimestamp;
const lastUpdatedMs = getLastUpdatedMs(dataset?.properties, dataset?.operations);
return (
<DatasetStatsSummary

13
datahub-web-react/src/app/entity/dataset/shared/utils.ts Normal file
View File

@ -0,0 +1,13 @@
import { DatasetProperties, Operation } from '../../../../types.generated';
export function getLastUpdatedMs(
properties: Pick<DatasetProperties, 'lastModified'> | null | undefined,
operations: Pick<Operation, 'lastUpdatedTimestamp'>[] | null | undefined,
): number | undefined {
return (
Math.max(
properties?.lastModified?.time || 0,
(operations?.length && operations[0].lastUpdatedTimestamp) || 0,
) || undefined
);
}

5
datahub-web-react/src/app/search/autoComplete/AutoCompleteTooltipContent.tsx
View File

@ -3,6 +3,7 @@ import React from 'react';
import styled from 'styled-components';
import { Dataset, Entity, EntityType } from '../../../types.generated';
import { DatasetStatsSummary } from '../../entity/dataset/shared/DatasetStatsSummary';
import { getLastUpdatedMs } from '../../entity/dataset/shared/utils';
import { useEntityRegistry } from '../../useEntityRegistry';
import { ArrowWrapper } from './ParentContainers';
@ -48,9 +49,7 @@ export default function AutoCompleteTooltipContent({ entity }: Props) {
rowCount={(entity as any).lastProfile?.length && (entity as any).lastProfile[0].rowCount}
columnCount={(entity as any).lastProfile?.length && (entity as any).lastProfile[0].columnCount}
sizeInBytes={(entity as any).lastProfile?.length && (entity as any).lastProfile[0].sizeInBytes}
lastUpdatedMs={
(entity as any).lastOperation?.length && (entity as any).lastOperation[0].lastUpdatedTimestamp
}
lastUpdatedMs={getLastUpdatedMs((entity as any)?.properties, (entity as any)?.lastOperation)}
queryCountLast30Days={(entity as Dataset).statsSummary?.queryCountLast30Days}
uniqueUserCountLast30Days={(entity as Dataset).statsSummary?.uniqueUserCountLast30Days}
mode="tooltip-content"

4
datahub-web-react/src/graphql/fragments.graphql
View File

@ -240,6 +240,10 @@ fragment nonRecursiveDatasetFields on Dataset {
value
}
externalUrl
lastModified {
time
actor
}
}
editableProperties {
description

12
datahub-web-react/src/graphql/search.graphql
View File

@ -13,6 +13,10 @@ fragment autoCompleteFields on Entity {
properties {
name
qualifiedName
lastModified {
time
actor
}
}
parentContainers {
...parentContainersFields
@ -39,6 +43,10 @@ fragment autoCompleteFields on Entity {
description
qualifiedName
externalUrl
lastModified {
time
actor
}
}
}
}
@ -336,6 +344,10 @@ fragment nonSiblingsDatasetSearchFields on Dataset {
value
}
externalUrl
lastModified {
time
actor
}
}
ownership {
...ownershipFields

38
metadata-ingestion/src/datahub/ingestion/source/unity/source.py
View File

@ -1,6 +1,5 @@
import logging
import re
import time
from concurrent.futures import ThreadPoolExecutor
from typing import Dict, Iterable, List, Optional, Set, Union
from urllib.parse import urljoin
@ -87,8 +86,6 @@ from datahub.metadata.schema_classes import (
DomainsClass,
MySqlDDLClass,
NullTypeClass,
OperationClass,
OperationTypeClass,
OwnerClass,
OwnershipClass,
OwnershipTypeClass,
@ -402,7 +399,6 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
sub_type = self._create_table_sub_type_aspect(table)
schema_metadata = self._create_schema_metadata_aspect(table)
operation = self._create_table_operation_aspect(table)
domain = self._get_domain_aspect(dataset_name=table.ref.qualified_table_name)
ownership = self._create_table_ownership_aspect(table)
data_platform_instance = self._create_data_platform_instance_aspect()
@ -424,7 +420,6 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
view_props,
sub_type,
schema_metadata,
operation,
domain,
ownership,
data_platform_instance,
@ -696,10 +691,10 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
int(table.created_at.timestamp() * 1000), make_user_urn(table.created_by)
)
last_modified = created
if table.updated_at and table.updated_by is not None:
if table.updated_at:
last_modified = TimeStampClass(
int(table.updated_at.timestamp() * 1000),
make_user_urn(table.updated_by),
table.updated_by and make_user_urn(table.updated_by),
)
return DatasetPropertiesClass(
@ -712,35 +707,6 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
externalUrl=f"{self.external_url_base}/{table.ref.external_path}",
)
def _create_table_operation_aspect(self, table: Table) -> OperationClass:
"""Produce an operation aspect for a table.
If a last updated time is present, we produce an update operation.
Otherwise, we produce a create operation. We do this in addition to
setting the last updated time in the dataset properties aspect, as
the UI is currently missing the ability to display the last updated
from the properties aspect.
"""
reported_time = int(time.time() * 1000)
operation = OperationClass(
timestampMillis=reported_time,
lastUpdatedTimestamp=int(table.created_at.timestamp() * 1000),
actor=make_user_urn(table.created_by),
operationType=OperationTypeClass.CREATE,
)
if table.updated_at and table.updated_by is not None:
operation = OperationClass(
timestampMillis=reported_time,
lastUpdatedTimestamp=int(table.updated_at.timestamp() * 1000),
actor=make_user_urn(table.updated_by),
operationType=OperationTypeClass.UPDATE,
)
return operation
def _create_table_ownership_aspect(self, table: Table) -> Optional[OwnershipClass]:
owner_urn = self.get_owner_urn(table.owner)
if owner_urn is not None:

207
metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json
View File

@ -524,29 +524,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.default.quickstart_table,PROD)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1638860400000,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"actor": "urn:li:corpuser:abc@acryl.io",
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1666186049633
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "unity-catalog-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.default.quickstart_table,PROD)",
@ -877,29 +854,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.information_schema.quickstart_table,PROD)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1638860400000,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"actor": "urn:li:corpuser:abc@acryl.io",
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1666186049633
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "unity-catalog-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.information_schema.quickstart_table,PROD)",
@ -1230,29 +1184,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.quickstart_schema.quickstart_table,PROD)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1638860400000,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"actor": "urn:li:corpuser:abc@acryl.io",
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1666186049633
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "unity-catalog-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.quickstart_schema.quickstart_table,PROD)",
@ -1719,29 +1650,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.default.quickstart_table,PROD)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1638860400000,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"actor": "urn:li:corpuser:abc@acryl.io",
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1666186049633
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "unity-catalog-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.default.quickstart_table,PROD)",
@ -2072,29 +1980,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.information_schema.quickstart_table,PROD)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1638860400000,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"actor": "urn:li:corpuser:abc@acryl.io",
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1666186049633
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "unity-catalog-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.information_schema.quickstart_table,PROD)",
@ -2425,29 +2310,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.quickstart_schema.quickstart_table,PROD)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1638860400000,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"actor": "urn:li:corpuser:abc@acryl.io",
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1666186049633
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "unity-catalog-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.quickstart_schema.quickstart_table,PROD)",
@ -2914,29 +2776,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.default.quickstart_table,PROD)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1638860400000,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"actor": "urn:li:corpuser:abc@acryl.io",
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1666186049633
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "unity-catalog-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.default.quickstart_table,PROD)",
@ -3267,29 +3106,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.information_schema.quickstart_table,PROD)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1638860400000,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"actor": "urn:li:corpuser:abc@acryl.io",
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1666186049633
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "unity-catalog-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.information_schema.quickstart_table,PROD)",
@ -3620,29 +3436,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.quickstart_schema.quickstart_table,PROD)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1638860400000,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"actor": "urn:li:corpuser:abc@acryl.io",
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1666186049633
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "unity-catalog-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.quickstart_schema.quickstart_table,PROD)",

2
metadata-ingestion/tests/unit/serde/test_serde.py
View File

@ -100,7 +100,7 @@ def test_serde_to_avro(
fo.seek(0)
in_records = list(fastavro.reader(fo, return_record_name=True))
in_mces = [
MetadataChangeEventClass.from_obj(record, tuples=True)
MetadataChangeEventClass.from_obj(record, tuples=True) # type: ignore
for record in in_records
]

2
metadata-service/openapi-entity-servlet/build.gradle
View File

@ -77,4 +77,4 @@ task openApiGenerate(type: GenerateSwaggerCode, dependsOn: [mergeApiComponents,
'delegatePattern' : "false"
]
}
tasks.getByName("compileJava").dependsOn(openApiGenerate)
tasks.getByName("compileJava").dependsOn(openApiGenerate)