diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Dataset/StatsSidebarSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Dataset/StatsSidebarSection.tsx index b6d9a5fb06..f7a7443756 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Dataset/StatsSidebarSection.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Dataset/StatsSidebarSection.tsx @@ -58,7 +58,7 @@ export const SidebarStatsSection = () => { const operations = (hasOperations && (baseEntity?.dataset?.operations as Array)) || undefined; const latestOperation = operations && operations[0]; - const lastUpdated = latestOperation && toLocalDateTimeString(latestOperation?.timestampMillis); + const lastUpdatedTime = latestOperation && toLocalDateTimeString(latestOperation?.lastUpdatedTimestamp); const routeToTab = useRouteToTab(); @@ -112,7 +112,7 @@ export const SidebarStatsSection = () => { onClick={() => routeToTab({ tabName: 'Queries' })} width={LAST_UPDATED_WIDTH_PX} > - {lastUpdated} + {lastUpdatedTime} ) : null} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/StatsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/StatsTab.tsx index e5b58ed573..1a220f66fd 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/StatsTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/StatsTab.tsx @@ -2,12 +2,7 @@ import React, { useState } from 'react'; import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated'; import { DatasetProfile, Operation, UsageQueryResult } from '../../../../../../types.generated'; import { useBaseEntity } from '../../../EntityContext'; -import { - toLocalDateString, - toLocalTimeString, - toLocalDateTimeString, - toUTCDateTimeString, -} from '../../../../../shared/time/timeUtils'; +import { toLocalDateString, toLocalTimeString, toLocalDateTimeString } from '../../../../../shared/time/timeUtils'; import HistoricalStats from './historical/HistoricalStats'; import { LOOKBACK_WINDOWS } from './lookbackWindows'; import ColumnStats from './snapshot/ColumnStats'; @@ -36,8 +31,8 @@ export default function StatsTab() { // Used for rendering operation info. const operations = (hasOperations && (baseEntity?.dataset?.operations as Array)) || undefined; const latestOperation = operations && operations[0]; - const lastUpdated = latestOperation && toLocalDateTimeString(latestOperation?.timestampMillis); - const lastUpdatedUTC = latestOperation && toUTCDateTimeString(latestOperation?.timestampMillis); + const lastUpdatedTime = latestOperation && toLocalDateTimeString(latestOperation?.lastUpdatedTimestamp); + const lastReportedTime = latestOperation && toLocalDateTimeString(latestOperation?.timestampMillis); // Okay so if we are disabled, we don't have both or the other. Let's render // const emptyView = ; @@ -68,8 +63,8 @@ export default function StatsTab() { columnCount={latestProfile?.columnCount || undefined} queryCount={usageStats?.aggregations?.totalSqlQueries || undefined} users={usageStats?.aggregations?.users || undefined} - lastUpdated={lastUpdated || undefined} - lastUpdatedUTC={lastUpdatedUTC || undefined} + lastUpdatedTime={lastUpdatedTime || undefined} + lastReportedTime={lastReportedTime || undefined} /> diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx index af8ab71bfd..f66fbd308c 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx @@ -12,8 +12,8 @@ type Props = { columnCount?: number; queryCount?: number; users?: Array>; - lastUpdated?: string; - lastUpdatedUTC?: string; + lastUpdatedTime?: string; + lastReportedTime?: string; }; const StatSection = styled.div` @@ -29,9 +29,17 @@ const StatContainer = styled.div<{ justifyContent }>` padding: 12px 2px; `; -export default function TableStats({ rowCount, columnCount, queryCount, users, lastUpdated, lastUpdatedUTC }: Props) { +export default function TableStats({ + rowCount, + columnCount, + queryCount, + users, + lastUpdatedTime, + lastReportedTime, +}: Props) { // If there are less than 4 items, simply stack the stat views. const justifyContent = !queryCount && !users ? 'default' : 'space-between'; + const lastReportedTimeString = lastReportedTime || 'unknown'; return ( Table Stats @@ -66,11 +74,11 @@ export default function TableStats({ rowCount, columnCount, queryCount, users, l )} - {lastUpdated && ( + {lastUpdatedTime && ( - + - {lastUpdated} + {lastUpdatedTime} diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index 333ed54ba0..db805d0549 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -120,6 +120,7 @@ query getDataset($urn: String!) { } operations(limit: 1) { timestampMillis + lastUpdatedTimestamp } upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { ...fullLineageResults diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/bigquery_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/bigquery_usage.py index 2427c172f5..41a4f03bb1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/bigquery_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/bigquery_usage.py @@ -6,6 +6,7 @@ import logging import os import re import textwrap +import time from dataclasses import dataclass, field from datetime import datetime from typing import Any, Dict, Iterable, List, MutableMapping, Optional, Union, cast @@ -925,6 +926,7 @@ class BigQueryUsageSource(Source): f"Failed to clean up destination table, {e}", ) return None + reported_time: int = int(time.time() * 1000) last_updated_timestamp: int = int(event.timestamp.timestamp() * 1000) affected_datasets = [] if event.referencedTables: @@ -942,7 +944,7 @@ class BigQueryUsageSource(Source): f"Failed to clean up table, {e}", ) operation_aspect = OperationClass( - timestampMillis=last_updated_timestamp, + timestampMillis=reported_time, lastUpdatedTimestamp=last_updated_timestamp, actor=builder.make_user_urn(event.actor_email.split("@")[0]), operationType=OPERATION_STATEMENT_TYPES[event.statementType], diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py index acdeb9f878..a8f01b25ff 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py @@ -1,6 +1,7 @@ import collections import dataclasses import logging +import time from datetime import datetime from typing import Dict, Iterable, List, Optional, Set @@ -191,6 +192,12 @@ class RedshiftUsageSource(Source): ::: + :::note + + Redshift system tables have some latency in getting data from queries. In addition, these tables only maintain logs for 2-5 days. You can find more information from the official documentation [here](https://aws.amazon.com/premiumsupport/knowledge-center/logs-redshift-database-cluster/). + + ::: + """ def __init__(self, config: RedshiftUsageConfig, ctx: PipelineContext): @@ -311,10 +318,11 @@ class RedshiftUsageSource(Source): assert event.operation_type in ["insert", "delete"] resource: str = f"{event.database}.{event.schema_}.{event.table}" + reported_time: int = int(time.time() * 1000) last_updated_timestamp: int = int(event.endtime.timestamp() * 1000) user_email: str = event.username operation_aspect = OperationClass( - timestampMillis=last_updated_timestamp, + timestampMillis=reported_time, lastUpdatedTimestamp=last_updated_timestamp, actor=builder.make_user_urn(user_email.split("@")[0]), operationType=( diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py index 410879c523..bdc183753d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py @@ -1,6 +1,7 @@ import collections import json import logging +import time from datetime import datetime, timezone from typing import Any, Dict, Iterable, List, Optional, Union, cast @@ -461,6 +462,7 @@ class SnowflakeUsageSource(StatefulIngestionSourceBase): query_type = event.query_type user_email = event.email operation_type = OPERATION_STATEMENT_TYPES[query_type] + reported_time: int = int(time.time() * 1000) last_updated_timestamp: int = int(start_time.timestamp() * 1000) user_urn = builder.make_user_urn(user_email.split("@")[0]) for obj in event.base_objects_accessed: @@ -472,7 +474,7 @@ class SnowflakeUsageSource(StatefulIngestionSourceBase): self.config.env, ) operation_aspect = OperationClass( - timestampMillis=last_updated_timestamp, + timestampMillis=reported_time, lastUpdatedTimestamp=last_updated_timestamp, actor=user_urn, operationType=operation_type, diff --git a/metadata-ingestion/tests/integration/bigquery-usage/bigquery_usages_golden.json b/metadata-ingestion/tests/integration/bigquery-usage/bigquery_usages_golden.json index 82b7a549c7..0026532fd6 100644 --- a/metadata-ingestion/tests/integration/bigquery-usage/bigquery_usages_golden.json +++ b/metadata-ingestion/tests/integration/bigquery-usage/bigquery_usages_golden.json @@ -7,7 +7,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1622073693816, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622073693816, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"]}", + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622073693816, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"]}", "contentType": "application/json" }, "systemMetadata": { @@ -26,7 +26,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1622074056868, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622074056868, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"]}", + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622074056868, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"]}", "contentType": "application/json" }, "systemMetadata": { @@ -45,7 +45,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1622075993220, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622075993220, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"]}", + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622075993220, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"]}", "contentType": "application/json" }, "systemMetadata": { @@ -64,7 +64,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1622076153701, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622076153701, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622076153701, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "contentType": "application/json" }, "systemMetadata": { @@ -83,7 +83,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1622076935475, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622076935475, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622076935475, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "contentType": "application/json" }, "systemMetadata": { @@ -102,7 +102,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1622091452779, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622091452779, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622091452779, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "contentType": "application/json" }, "systemMetadata": { @@ -121,7 +121,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1622098252897, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622098252897, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622098252897, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "contentType": "application/json" }, "systemMetadata": { @@ -140,7 +140,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1622148197222, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622148197222, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622148197222, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "contentType": "application/json" }, "systemMetadata": { @@ -159,7 +159,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1622148734552, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622148734552, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622148734552, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "contentType": "application/json" }, "systemMetadata": { @@ -178,7 +178,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1622161940000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622161940000, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622161940000, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "contentType": "application/json" }, "systemMetadata": { @@ -197,7 +197,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1622243780153, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"]}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/redshift-usage/redshift_usages_filtered_golden.json b/metadata-ingestion/tests/integration/redshift-usage/redshift_usages_filtered_golden.json index 4faa0dea5a..68c8deda88 100644 --- a/metadata-ingestion/tests/integration/redshift-usage/redshift_usages_filtered_golden.json +++ b/metadata-ingestion/tests/integration/redshift-usage/redshift_usages_filtered_golden.json @@ -7,7 +7,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1631664000000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:test-name\", \"operationType\": \"INSERT\"}", + "value": "{\"timestampMillis\": 1629795600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:test-name\", \"operationType\": \"INSERT\"}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/redshift-usage/redshift_usages_golden.json b/metadata-ingestion/tests/integration/redshift-usage/redshift_usages_golden.json index 44516b241a..dee16ec7eb 100644 --- a/metadata-ingestion/tests/integration/redshift-usage/redshift_usages_golden.json +++ b/metadata-ingestion/tests/integration/redshift-usage/redshift_usages_golden.json @@ -7,7 +7,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1631664000000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:test-name\", \"operationType\": \"INSERT\"}", + "value": "{\"timestampMillis\": 1629795600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:test-name\", \"operationType\": \"INSERT\"}", "contentType": "application/json" }, "systemMetadata": { @@ -26,7 +26,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1631664000000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:real_shirshanka\", \"operationType\": \"INSERT\"}", + "value": "{\"timestampMillis\": 1629795600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:real_shirshanka\", \"operationType\": \"INSERT\"}", "contentType": "application/json" }, "systemMetadata": { @@ -45,7 +45,7 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1631664000000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:real_shirshanka\", \"operationType\": \"DELETE\"}", + "value": "{\"timestampMillis\": 1629795600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"lastUpdatedTimestamp\": 1631664000000, \"actor\": \"urn:li:corpuser:real_shirshanka\", \"operationType\": \"DELETE\"}", "contentType": "application/json" }, "systemMetadata": {