yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-07-27 11:29:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(operation): display the reported time for last updated in the UI (#4800)

Browse Source
This commit is contained in:
Aditya Radhakrishnan 2022-05-02 16:00:29 -07:00 committed by GitHub
parent e3eb4b190b
commit c20a47f34c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 52 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Dataset/StatsSidebarSection.tsx
View File

@ -58,7 +58,7 @@ export const SidebarStatsSection = () => {
const operations = (hasOperations && (baseEntity?.dataset?.operations as Array<Operation>)) || undefined; const operations = (hasOperations && (baseEntity?.dataset?.operations as Array<Operation>)) || undefined;
const latestOperation = operations && operations[0]; const latestOperation = operations && operations[0];
const lastUpdated = latestOperation && toLocalDateTimeString(latestOperation?.timestampMillis); const lastUpdatedTime = latestOperation && toLocalDateTimeString(latestOperation?.lastUpdatedTimestamp);
const routeToTab = useRouteToTab(); const routeToTab = useRouteToTab();
@ -112,7 +112,7 @@ export const SidebarStatsSection = () => {
onClick={() => routeToTab({ tabName: 'Queries' })} onClick={() => routeToTab({ tabName: 'Queries' })}
width={LAST_UPDATED_WIDTH_PX} width={LAST_UPDATED_WIDTH_PX}
> >
<HeaderInfoBody>{lastUpdated}</HeaderInfoBody> <HeaderInfoBody>{lastUpdatedTime}</HeaderInfoBody>
</InfoItem> </InfoItem>
) : null} ) : null}
</StatsRow> </StatsRow>

15
datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/StatsTab.tsx
View File

@ -2,12 +2,7 @@ import React, { useState } from 'react';
import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated'; import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated';
import { DatasetProfile, Operation, UsageQueryResult } from '../../../../../../types.generated'; import { DatasetProfile, Operation, UsageQueryResult } from '../../../../../../types.generated';
import { useBaseEntity } from '../../../EntityContext'; import { useBaseEntity } from '../../../EntityContext';
import { import { toLocalDateString, toLocalTimeString, toLocalDateTimeString } from '../../../../../shared/time/timeUtils';
toLocalDateString,
toLocalTimeString,
toLocalDateTimeString,
toUTCDateTimeString,
} from '../../../../../shared/time/timeUtils';
import HistoricalStats from './historical/HistoricalStats'; import HistoricalStats from './historical/HistoricalStats';
import { LOOKBACK_WINDOWS } from './lookbackWindows'; import { LOOKBACK_WINDOWS } from './lookbackWindows';
import ColumnStats from './snapshot/ColumnStats'; import ColumnStats from './snapshot/ColumnStats';
@ -36,8 +31,8 @@ export default function StatsTab() {
// Used for rendering operation info. // Used for rendering operation info.
const operations = (hasOperations && (baseEntity?.dataset?.operations as Array<Operation>)) || undefined; const operations = (hasOperations && (baseEntity?.dataset?.operations as Array<Operation>)) || undefined;
const latestOperation = operations && operations[0]; const latestOperation = operations && operations[0];
const lastUpdated = latestOperation && toLocalDateTimeString(latestOperation?.timestampMillis); const lastUpdatedTime = latestOperation && toLocalDateTimeString(latestOperation?.lastUpdatedTimestamp);
const lastUpdatedUTC = latestOperation && toUTCDateTimeString(latestOperation?.timestampMillis); const lastReportedTime = latestOperation && toLocalDateTimeString(latestOperation?.timestampMillis);
// Okay so if we are disabled, we don't have both or the other. Let's render // Okay so if we are disabled, we don't have both or the other. Let's render
// const emptyView = <Empty description="TODO: Stats!" image={Empty.PRESENTED_IMAGE_SIMPLE} />; // const emptyView = <Empty description="TODO: Stats!" image={Empty.PRESENTED_IMAGE_SIMPLE} />;
@ -68,8 +63,8 @@ export default function StatsTab() {
columnCount={latestProfile?.columnCount || undefined} columnCount={latestProfile?.columnCount || undefined}
queryCount={usageStats?.aggregations?.totalSqlQueries || undefined} queryCount={usageStats?.aggregations?.totalSqlQueries || undefined}
users={usageStats?.aggregations?.users || undefined} users={usageStats?.aggregations?.users || undefined}
lastUpdated={lastUpdated || undefined} lastUpdatedTime={lastUpdatedTime || undefined}
lastUpdatedUTC={lastUpdatedUTC || undefined} lastReportedTime={lastReportedTime || undefined}
/> />
<ColumnStats columnStats={(latestProfile && latestProfile.fieldProfiles) || []} /> <ColumnStats columnStats={(latestProfile && latestProfile.fieldProfiles) || []} />
</> </>

20
datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx
View File

@ -12,8 +12,8 @@ type Props = {
columnCount?: number; columnCount?: number;
queryCount?: number; queryCount?: number;
users?: Array<Maybe<UserUsageCounts>>; users?: Array<Maybe<UserUsageCounts>>;
lastUpdated?: string; lastUpdatedTime?: string;
lastUpdatedUTC?: string; lastReportedTime?: string;
}; };
const StatSection = styled.div` const StatSection = styled.div`
@ -29,9 +29,17 @@ const StatContainer = styled.div<{ justifyContent }>`
padding: 12px 2px; padding: 12px 2px;
`; `;
export default function TableStats({ rowCount, columnCount, queryCount, users, lastUpdated, lastUpdatedUTC }: Props) { export default function TableStats({
rowCount,
columnCount,
queryCount,
users,
lastUpdatedTime,
lastReportedTime,
}: Props) {
// If there are less than 4 items, simply stack the stat views. // If there are less than 4 items, simply stack the stat views.
const justifyContent = !queryCount && !users ? 'default' : 'space-between'; const justifyContent = !queryCount && !users ? 'default' : 'space-between';
const lastReportedTimeString = lastReportedTime || 'unknown';
return ( return (
<StatSection> <StatSection>
<Typography.Title level={5}>Table Stats</Typography.Title> <Typography.Title level={5}>Table Stats</Typography.Title>
@ -66,11 +74,11 @@ export default function TableStats({ rowCount, columnCount, queryCount, users, l
</div> </div>
</InfoItem> </InfoItem>
)} )}
{lastUpdated && ( {lastUpdatedTime && (
<InfoItem title="Last Updated" width="220px"> <InfoItem title="Last Updated" width="220px">
<Tooltip title={lastUpdatedUTC}> <Tooltip title={`Last reported at ${lastReportedTimeString}`}>
<Typography.Text strong style={{ fontSize: 16 }}> <Typography.Text strong style={{ fontSize: 16 }}>
{lastUpdated} {lastUpdatedTime}
</Typography.Text> </Typography.Text>
</Tooltip> </Tooltip>
</InfoItem> </InfoItem>

1
datahub-web-react/src/graphql/dataset.graphql
View File

@ -120,6 +120,7 @@ query getDataset($urn: String!) {
} }
operations(limit: 1) { operations(limit: 1) {
timestampMillis timestampMillis
lastUpdatedTimestamp
} }
upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) {
...fullLineageResults ...fullLineageResults

4
metadata-ingestion/src/datahub/ingestion/source/usage/bigquery_usage.py
View File

@ -6,6 +6,7 @@ import logging
import os import os
import re import re
import textwrap import textwrap
import time
from dataclasses import dataclass, field from dataclasses import dataclass, field
from datetime import datetime from datetime import datetime
from typing import Any, Dict, Iterable, List, MutableMapping, Optional, Union, cast from typing import Any, Dict, Iterable, List, MutableMapping, Optional, Union, cast
@ -925,6 +926,7 @@ class BigQueryUsageSource(Source):
f"Failed to clean up destination table, {e}", f"Failed to clean up destination table, {e}",
) )
return None return None
reported_time: int = int(time.time() * 1000)
last_updated_timestamp: int = int(event.timestamp.timestamp() * 1000) last_updated_timestamp: int = int(event.timestamp.timestamp() * 1000)
affected_datasets = [] affected_datasets = []
if event.referencedTables: if event.referencedTables:
@ -942,7 +944,7 @@ class BigQueryUsageSource(Source):
f"Failed to clean up table, {e}", f"Failed to clean up table, {e}",
) )
operation_aspect = OperationClass( operation_aspect = OperationClass(
timestampMillis=last_updated_timestamp, timestampMillis=reported_time,
lastUpdatedTimestamp=last_updated_timestamp, lastUpdatedTimestamp=last_updated_timestamp,
actor=builder.make_user_urn(event.actor_email.split("@")[0]), actor=builder.make_user_urn(event.actor_email.split("@")[0]),
operationType=OPERATION_STATEMENT_TYPES[event.statementType], operationType=OPERATION_STATEMENT_TYPES[event.statementType],

10
metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py
View File

@ -1,6 +1,7 @@
import collections import collections
import dataclasses import dataclasses
import logging import logging
import time
from datetime import datetime from datetime import datetime
from typing import Dict, Iterable, List, Optional, Set from typing import Dict, Iterable, List, Optional, Set
@ -191,6 +192,12 @@ class RedshiftUsageSource(Source):
::: :::
:::note
Redshift system tables have some latency in getting data from queries. In addition, these tables only maintain logs for 2-5 days. You can find more information from the official documentation [here](https://aws.amazon.com/premiumsupport/knowledge-center/logs-redshift-database-cluster/).
:::
""" """
def __init__(self, config: RedshiftUsageConfig, ctx: PipelineContext): def __init__(self, config: RedshiftUsageConfig, ctx: PipelineContext):
@ -311,10 +318,11 @@ class RedshiftUsageSource(Source):
assert event.operation_type in ["insert", "delete"] assert event.operation_type in ["insert", "delete"]
resource: str = f"{event.database}.{event.schema_}.{event.table}" resource: str = f"{event.database}.{event.schema_}.{event.table}"
reported_time: int = int(time.time() * 1000)
last_updated_timestamp: int = int(event.endtime.timestamp() * 1000) last_updated_timestamp: int = int(event.endtime.timestamp() * 1000)
user_email: str = event.username user_email: str = event.username
operation_aspect = OperationClass( operation_aspect = OperationClass(
timestampMillis=last_updated_timestamp, timestampMillis=reported_time,
lastUpdatedTimestamp=last_updated_timestamp, lastUpdatedTimestamp=last_updated_timestamp,
actor=builder.make_user_urn(user_email.split("@")[0]), actor=builder.make_user_urn(user_email.split("@")[0]),
operationType=( operationType=(

4
metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py
View File

@ -1,6 +1,7 @@
import collections import collections
import json import json
import logging import logging
import time
from datetime import datetime, timezone from datetime import datetime, timezone
from typing import Any, Dict, Iterable, List, Optional, Union, cast from typing import Any, Dict, Iterable, List, Optional, Union, cast
@ -461,6 +462,7 @@ class SnowflakeUsageSource(StatefulIngestionSourceBase):
query_type = event.query_type query_type = event.query_type
user_email = event.email user_email = event.email
operation_type = OPERATION_STATEMENT_TYPES[query_type] operation_type = OPERATION_STATEMENT_TYPES[query_type]
reported_time: int = int(time.time() * 1000)
last_updated_timestamp: int = int(start_time.timestamp() * 1000) last_updated_timestamp: int = int(start_time.timestamp() * 1000)
user_urn = builder.make_user_urn(user_email.split("@")[0]) user_urn = builder.make_user_urn(user_email.split("@")[0])
for obj in event.base_objects_accessed: for obj in event.base_objects_accessed:
@ -472,7 +474,7 @@ class SnowflakeUsageSource(StatefulIngestionSourceBase):
self.config.env, self.config.env,
) )
operation_aspect = OperationClass( operation_aspect = OperationClass(
timestampMillis=last_updated_timestamp, timestampMillis=reported_time,
lastUpdatedTimestamp=last_updated_timestamp, lastUpdatedTimestamp=last_updated_timestamp,
actor=user_urn, actor=user_urn,
operationType=operation_type, operationType=operation_type,

22
metadata-ingestion/tests/integration/bigquery-usage/bigquery_usages_golden.json
View File

@ -7,7 +7,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1622073693816, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622073693816, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"]}", "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622073693816, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"]}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -26,7 +26,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1622074056868, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622074056868, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"]}", "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622074056868, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"]}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -45,7 +45,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1622075993220, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622075993220, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"]}", "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622075993220, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"]}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -64,7 +64,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1622076153701, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622076153701, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622076153701, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -83,7 +83,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1622076935475, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622076935475, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622076935475, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -102,7 +102,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1622091452779, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622091452779, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622091452779, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -121,7 +121,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1622098252897, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622098252897, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622098252897, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -140,7 +140,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1622148197222, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622148197222, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622148197222, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -159,7 +159,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1622148734552, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622148734552, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622148734552, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -178,7 +178,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1622161940000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622161940000, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622161940000, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -197,7 +197,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1622243780153, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {

2
metadata-ingestion/tests/integration/redshift-usage/redshift_usages_filtered_golden.json
View File

@ -7,7 +7,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1631664000000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:test-name\", \"operationType\": \"INSERT\"}", "value": "{\"timestampMillis\": 1629795600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:test-name\", \"operationType\": \"INSERT\"}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {

6
metadata-ingestion/tests/integration/redshift-usage/redshift_usages_golden.json
View File

@ -7,7 +7,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1631664000000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:test-name\", \"operationType\": \"INSERT\"}", "value": "{\"timestampMillis\": 1629795600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:test-name\", \"operationType\": \"INSERT\"}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -26,7 +26,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1631664000000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:real_shirshanka\", \"operationType\": \"INSERT\"}", "value": "{\"timestampMillis\": 1629795600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:real_shirshanka\", \"operationType\": \"INSERT\"}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -45,7 +45,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "operation", "aspectName": "operation",
"aspect": { "aspect": {
"value": "{\"timestampMillis\": 1631664000000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:real_shirshanka\", \"operationType\": \"DELETE\"}", "value": "{\"timestampMillis\": 1629795600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:real_shirshanka\", \"operationType\": \"DELETE\"}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {