yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-02 03:29:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

FIX - Query checksum & fix update (#16392)

Browse Source 
* FIX - Query checksum & fix update

* FIX - Query checksum & fix update

* FIX - Query checksum & fix update

* FIX - Query checksum & fix update

* FIX - Query checksum & fix update
This commit is contained in:
Pere Miquel Brull 2024-05-22 20:19:42 +02:00 committed by GitHub
parent 3a842f58e3
commit c33d1bbb18
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 58 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
bootstrap/sql/migrations/native/1.4.0/mysql/schemaChanges.sql
View File

@ -3,11 +3,38 @@ UPDATE dbservice_entity
SET json = JSON_INSERT(json, '$.connection.config.supportsProfiler', TRUE)
WHERE serviceType = 'MongoDB';
-- Queries should be unique:
-- 1. Remove duplicate queries from entity_relationship
-- 2. Remove duplicate queries from query_entity
-- 3. Add checksum with unique constraint
ALTER TABLE query_entity ADD COLUMN checksum VARCHAR
(32) GENERATED ALWAYS AS
(json ->> '$.checksum') NOT NULL UNIQUE;
(json ->> '$.checksum') NOT NULL;
UPDATE query_entity SET json = JSON_INSERT(json, '$.checksum', MD5(JSON_UNQUOTE(JSON_EXTRACT(json, '$.checksum'))));
with duplicated as (
select
id,
ROW_NUMBER() OVER (PARTITION BY checksum ORDER BY id) AS rn
FROM query_entity
)
DELETE FROM entity_relationship
where toEntity = 'query' and toId in (
select id from duplicated where rn > 1
);
with duplicated as (
select
id,
ROW_NUMBER() OVER (PARTITION BY checksum ORDER BY id) AS rn
FROM query_entity
)
DELETE FROM query_entity where id in (
select id from duplicated where rn > 1
);
ALTER TABLE query_entity ADD CONSTRAINT unique_query_checksum UNIQUE (checksum);
UPDATE query_entity SET json = JSON_INSERT(json, '$.checksum', MD5(JSON_UNQUOTE(checksum)));
-- Restructure dbServiceNames in ingestion_pipeline_entity
update ingestion_pipeline_entity set json =

33
bootstrap/sql/migrations/native/1.4.0/postgres/schemaChanges.sql
View File

@ -4,13 +4,38 @@ SET json = jsonb_set(json::jsonb, '{connection,config,supportsProfiler}', 'true'
::jsonb)
WHERE serviceType = 'MongoDB';
-- Queries should be unique:
-- 1. Remove duplicate queries from entity_relationship
-- 2. Remove duplicate queries from query_entity
-- 3. Add checksum with unique constraint
ALTER TABLE query_entity ADD COLUMN checksum varchar
(32) GENERATED ALWAYS AS
(json ->> 'checksum') STORED NOT NULL,
ADD UNIQUE
(checksum);
(json ->> 'checksum') STORED NOT NULL;
UPDATE query_entity SET json = jsonb_set(json::jsonb, '{checksum}', MD5((json->>'checksum')::text)::jsonb);
with duplicated as (
select
id,
ROW_NUMBER() OVER (PARTITION BY checksum ORDER BY id) AS rn
FROM query_entity
)
DELETE FROM entity_relationship
where toEntity = 'query' and toId in (
select id from duplicated where rn > 1
);
with duplicated as (
select
id,
ROW_NUMBER() OVER (PARTITION BY checksum ORDER BY id) AS rn
FROM query_entity
)
DELETE FROM query_entity where id in (
select id from duplicated where rn > 1
);
ALTER TABLE query_entity ADD CONSTRAINT unique_query_checksum UNIQUE (checksum);
UPDATE query_entity SET json = jsonb_set(json::jsonb, '{checksum}', to_jsonb(MD5(checksum)));
-- Restructure dbServiceNames in ingestion_pipeline_entity
update ingestion_pipeline_entity ipe