mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-08-06 08:08:10 +00:00
ab3042e8ee
* fix: updated ingestion to delete existing data for the ingestion day * fix: added delete endpoint for the report data * fix: added migration to delete duplicate data + added tables creation & data deltion logic in a transaction * fix: made deletion SQL engine aware * fix: added ES deletion logic back while we work on DI event publisher
53 lines
1.6 KiB
PL/PgSQL
53 lines
1.6 KiB
PL/PgSQL
BEGIN;
|
|
-- We'll rank all the runs (timestamps) for every day, and delete all the data but the most recent one.
|
|
DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
|
|
select ids FROM (
|
|
SELECT
|
|
(json ->> 'id') AS ids,
|
|
DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
|
|
FROM (
|
|
SELECT
|
|
*,
|
|
DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
|
|
FROM report_data_time_series rdts
|
|
WHERE json ->> 'reportDataType' = 'WebAnalyticEntityViewReportData'
|
|
) duplicates
|
|
ORDER BY date DESC, timestamp DESC
|
|
) as dense_ranked
|
|
WHERE denseRank != 1
|
|
);
|
|
|
|
DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
|
|
select ids FROM (
|
|
SELECT
|
|
(json ->> 'id') AS ids,
|
|
DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
|
|
FROM (
|
|
SELECT
|
|
*,
|
|
DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
|
|
FROM report_data_time_series rdts
|
|
WHERE json ->> 'reportDataType' = 'EntityReportData'
|
|
) duplicates
|
|
ORDER BY date DESC, timestamp DESC
|
|
) as dense_ranked
|
|
WHERE denseRank != 1
|
|
);
|
|
|
|
DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
|
|
select ids FROM (
|
|
SELECT
|
|
(json ->> 'id') AS ids,
|
|
DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
|
|
FROM (
|
|
SELECT
|
|
*,
|
|
DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
|
|
FROM report_data_time_series rdts
|
|
WHERE json ->> 'reportDataType' = 'WebAnalyticUserActivityReportData'
|
|
) duplicates
|
|
ORDER BY date DESC, timestamp DESC
|
|
) as dense_ranked
|
|
WHERE denseRank != 1
|
|
);
|
|
COMMIT; |