yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-26 15:10:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #4448 Consolidate different versions of bootstrap sql files (#4450)

Browse Source
This commit is contained in:
Suresh Srinivas 2022-04-24 13:16:24 -07:00 committed by GitHub
parent feafa448e6
commit e880224da8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 132 additions and 797 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

258
bootstrap/sql/mysql/v001__create_db_connection_info.sql
View File

@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS entity_relationship (
relation TINYINT NOT NULL,
jsonSchema VARCHAR(256), -- Schema used for generating JSON
json JSON, -- JSON payload with additional information
timestamp BIGINT,
deleted BOOLEAN NOT NULL DEFAULT 0,
INDEX edgeIdx (fromId, toId, relation),
INDEX fromIdx (fromId, relation),
INDEX toIdx (toId, relation),
@ -29,8 +29,6 @@ CREATE TABLE IF NOT EXISTS field_relationship (
relation TINYINT NOT NULL,
jsonSchema VARCHAR(256), -- Schema used for generating JSON
json JSON, -- JSON payload with additional information
timestamp BIGINT,
INDEX edgeIdx (fromFQN, toFQN, relation),
INDEX fromIdx (fromFQN, relation),
INDEX toIdx (toFQN, relation),
PRIMARY KEY (fromFQN, toFQN, relation)
@ -44,7 +42,6 @@ CREATE TABLE IF NOT EXISTS entity_extension (
extension VARCHAR(256) NOT NULL, -- Extension name same as entity.fieldName
jsonSchema VARCHAR(256) NOT NULL, -- Schema used for generating JSON
json JSON NOT NULL,
timestamp BIGINT,
PRIMARY KEY (id, extension)
);
@ -56,13 +53,11 @@ CREATE TABLE IF NOT EXISTS dbservice_entity (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(name)
);
CREATE TABLE IF NOT EXISTS messaging_service_entity (
@ -70,13 +65,11 @@ CREATE TABLE IF NOT EXISTS messaging_service_entity (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(name)
);
CREATE TABLE IF NOT EXISTS dashboard_service_entity (
@ -84,13 +77,11 @@ CREATE TABLE IF NOT EXISTS dashboard_service_entity (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(name)
);
CREATE TABLE IF NOT EXISTS pipeline_service_entity (
@ -98,13 +89,11 @@ CREATE TABLE IF NOT EXISTS pipeline_service_entity (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(name)
);
CREATE TABLE IF NOT EXISTS storage_service_entity (
@ -112,7 +101,9 @@ CREATE TABLE IF NOT EXISTS storage_service_entity (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
json JSON NOT NULL,
timestamp BIGINT,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
);
@ -124,136 +115,119 @@ CREATE TABLE IF NOT EXISTS database_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS database_schema_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS table_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
);
CREATE TABLE IF NOT EXISTS dbt_model_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS metric_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS report_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS dashboard_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS ml_model_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS pipeline_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS topic_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS chart_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS location_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
timestamp BIGINT,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL ,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
);
@ -263,8 +237,15 @@ CREATE TABLE IF NOT EXISTS location_entity (
--
CREATE TABLE IF NOT EXISTS thread_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
entityId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityId') NOT NULL,
entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.about') NOT NULL,
assignedTo VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.addressedTo'),
json JSON NOT NULL,
timestamp BIGINT,
createdAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.threadTs') STORED NOT NULL,
createdBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.createdBy') STORED NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
resolved BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.resolved')),
PRIMARY KEY (id)
);
@ -275,29 +256,26 @@ CREATE TABLE IF NOT EXISTS policy_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(fullyQualifiedName)
);
--
-- Ingestion related tables
--
CREATE TABLE IF NOT EXISTS ingestion_entity (
CREATE TABLE IF NOT EXISTS ingestion_pipeline_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
timestamp BIGINT,
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(fullyQualifiedName)
);
--
@ -307,13 +285,11 @@ CREATE TABLE IF NOT EXISTS team_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(name)
);
CREATE TABLE IF NOT EXISTS user_entity (
@ -322,33 +298,32 @@ CREATE TABLE IF NOT EXISTS user_entity (
email VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.email') NOT NULL,
deactivated VARCHAR(8) GENERATED ALWAYS AS (json ->> '$.deactivated'),
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(name)
);
CREATE TABLE IF NOT EXISTS bot_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
json JSON NOT NULL,
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name),
INDEX (updatedBy),
INDEX (updatedAt)
UNIQUE KEY unique_name(name)
);
CREATE TABLE IF NOT EXISTS role_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
json JSON NOT NULL,
timestamp BIGINT,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
defaultRole BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.defaultRole')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
);
@ -373,43 +348,74 @@ CREATE TABLE IF NOT EXISTS entity_usage (
-- Tag related tables
--
CREATE TABLE IF NOT EXISTS tag_category (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
json JSON NOT NULL, -- JSON stores category information and does not store children
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
UNIQUE KEY unique_name(name), -- Unique tag category name
INDEX (updatedBy),
INDEX (updatedAt)
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
UNIQUE KEY unique_name(name) -- Unique tag category name
);
CREATE TABLE IF NOT EXISTS tag (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL, -- JSON stores all tag attributes and does not store children
updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
timestamp BIGINT,
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
UNIQUE KEY unique_name(fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS tag_usage (
source TINYINT NOT NULL, -- Source of the tag label
tagFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the tag
targetFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the entity instance or corresponding field
labelType TINYINT NOT NULL, -- Type of tagging: manual, automated, propagated, derived
state TINYINT NOT NULL, -- State of tagging: suggested or confirmed
timestamp BIGINT,
UNIQUE KEY unique_name(tagFQN, targetFQN)
UNIQUE KEY unique_name(source, tagFQN, targetFQN)
);
CREATE TABLE IF NOT EXISTS change_event (
eventType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.eventType') NOT NULL,
entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityType') NOT NULL,
userName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.userName') NOT NULL,
dateTime TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.dateTime', '%Y-%m-%dT%T.%fZ'))) NOT NULL,
eventTime BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
json JSON NOT NULL,
INDEX (dateTime),
INDEX (eventType),
INDEX (entityType)
);
INDEX (entityType),
INDEX (eventTime)
);
CREATE TABLE IF NOT EXISTS webhook_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
json JSON NOT NULL,
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
-- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
);
CREATE TABLE IF NOT EXISTS glossary_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
);
CREATE TABLE IF NOT EXISTS glossary_term_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
);

53
bootstrap/sql/mysql/v002__create_db_connection_info.sql
View File

@ -1,53 +0,0 @@
-- DBT models are no longer a separate entity
DROP TABLE IF EXISTS dbt_model_entity;
DELETE FROM entity_relationship WHERE fromEntity='dbtmodel' OR toEntity='dbtmodel';
--
-- Add updatedAt and updatedBy columns for entities that are missing it and index those columns
-- Drop timestamp column from all the tables as they are not used
--
ALTER TABLE location_entity
DROP COLUMN timestamp,
ADD COLUMN updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL AFTER json,
ADD COLUMN updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL AFTER updatedAt,
ADD INDEX (updatedBy),
ADD INDEX (updatedAt);
ALTER TABLE storage_service_entity
DROP COLUMN timestamp,
ADD COLUMN updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL AFTER json,
ADD COLUMN updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL AFTER updatedAt,
ADD INDEX (updatedBy),
ADD INDEX (updatedAt);
ALTER TABLE role_entity
DROP COLUMN timestamp,
ADD COLUMN updatedAt TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ'))) NOT NULL AFTER json,
ADD COLUMN updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL AFTER updatedAt,
ADD INDEX (updatedBy),
ADD INDEX (updatedAt);
ALTER TABLE entity_relationship DROP COLUMN timestamp;
ALTER TABLE field_relationship DROP COLUMN timestamp;
ALTER TABLE entity_extension DROP COLUMN timestamp;
ALTER TABLE dbservice_entity DROP COLUMN timestamp;
ALTER TABLE messaging_service_entity DROP COLUMN timestamp;
ALTER TABLE dashboard_service_entity DROP COLUMN timestamp;
ALTER TABLE pipeline_service_entity DROP COLUMN timestamp;
ALTER TABLE database_entity DROP COLUMN timestamp;
ALTER TABLE metric_entity DROP COLUMN timestamp;
ALTER TABLE report_entity DROP COLUMN timestamp;
ALTER TABLE dashboard_entity DROP COLUMN timestamp;
ALTER TABLE ml_model_entity DROP COLUMN timestamp;
ALTER TABLE pipeline_entity DROP COLUMN timestamp;
ALTER TABLE topic_entity DROP COLUMN timestamp;
ALTER TABLE chart_entity DROP COLUMN timestamp;
ALTER TABLE thread_entity DROP COLUMN timestamp;
ALTER TABLE policy_entity DROP COLUMN timestamp;
ALTER TABLE ingestion_entity DROP COLUMN timestamp;
ALTER TABLE team_entity DROP COLUMN timestamp;
ALTER TABLE user_entity DROP COLUMN timestamp;
ALTER TABLE bot_entity DROP COLUMN timestamp;
ALTER TABLE tag_category DROP COLUMN timestamp;
ALTER TABLE tag DROP COLUMN timestamp;
ALTER TABLE tag_usage DROP COLUMN timestamp;

387
bootstrap/sql/mysql/v003__create_db_connection_info.sql
View File

@ -1,387 +0,0 @@
CREATE TABLE IF NOT EXISTS webhook_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
json JSON NOT NULL,
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
-- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
);
UPDATE change_event
SET json = JSON_SET(json, '$.timestamp', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.dateTime', '%Y-%m-%dT%T.%fZ')));
ALTER TABLE change_event
DROP INDEX dateTime,
DROP COLUMN dateTime,
ADD COLUMN eventTime BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL AFTER username,
ADD INDEX (eventTime);
UPDATE change_event
SET json = JSON_REMOVE(json, '$.dateTime');
--
-- Update to add deleted fields to data entities and change updatedAt field to unix epoch time milliseconds
--
ALTER TABLE dbservice_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE dbservice_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
UPDATE dbservice_entity
SET json = JSON_REMOVE(json, '$.jdbc');
ALTER TABLE dbservice_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE messaging_service_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE messaging_service_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE messaging_service_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE dashboard_service_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE dashboard_service_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE dashboard_service_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE pipeline_service_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE pipeline_service_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE pipeline_service_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE storage_service_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE storage_service_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE storage_service_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE database_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE database_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE database_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE table_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE table_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE table_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE metric_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE metric_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE metric_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE report_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE report_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE report_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE dashboard_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE dashboard_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE dashboard_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE ml_model_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE ml_model_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE ml_model_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE pipeline_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE pipeline_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE pipeline_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE topic_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE topic_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE, '$.retentionSize', 2147483647);
ALTER TABLE topic_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE chart_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE chart_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE chart_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE location_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE location_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE location_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE bot_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE bot_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE bot_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE policy_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE policy_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE policy_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
-- Update entity extension data where we store versions of entities which will have updatedAt in old format.
UPDATE entity_extension
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')))
where extension like '%.version.%';
ALTER TABLE ingestion_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE ingestion_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE ingestion_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE team_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE team_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE team_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE role_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE role_entity
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE role_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE tag_category
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE tag_category
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE tag_category
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE tag
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE tag
SET json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE tag
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE entity_relationship
ADD COLUMN deleted BOOLEAN NOT NULL DEFAULT 0,
ADD INDEX (deleted);
-- Change "team -- contains --> user" relationship to "team -- has --> user" relationship
UPDATE entity_relationship
SET relation = 10 WHERE fromEntity = 'team' AND toEntity = 'user' AND relation = 0;
-- Change "dashboard -- contains --> chart" relationship to "dashboard -- has --> chart" relationship
UPDATE entity_relationship
SET relation = 10 WHERE fromEntity = 'dashboard' AND toEntity = 'chart' AND relation = 0;
-- Remove user.deactivated field and use deleted instead
ALTER TABLE user_entity
DROP COLUMN updatedAt,
DROP INDEX updatedAt;
UPDATE user_entity
SET json = JSON_REMOVE(user_entity.json, '$.deactivated'),
json = JSON_SET(json, '$.updatedAt', UNIX_TIMESTAMP(STR_TO_DATE(json ->> '$.updatedAt', '%Y-%m-%dT%T.%fZ')), '$.deleted', FALSE);
ALTER TABLE user_entity
ADD COLUMN updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL AFTER json,
ADD INDEX(updatedAt),
DROP COLUMN deactivated,
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
-- Rename airflow pipeline entities
--
-- Ingestion related tables
--
DROP TABLE IF EXISTS ingestion_entity;
CREATE TABLE IF NOT EXISTS airflow_pipeline_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
timestamp BIGINT,
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
);

68
bootstrap/sql/mysql/v004__create_db_connection_info.sql
View File

@ -1,68 +0,0 @@
DROP TABLE IF EXISTS thread_entity;
CREATE TABLE IF NOT EXISTS thread_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
entityId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityId') NOT NULL,
entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.about') NOT NULL,
assignedTo VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.addressedTo'),
json JSON NOT NULL,
createdAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.threadTs') STORED NOT NULL,
createdBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.createdBy') STORED NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
resolved BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.resolved')),
PRIMARY KEY (id),
INDEX (updatedBy),
INDEX (updatedAt)
);
CREATE TABLE IF NOT EXISTS glossary_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name),
INDEX (updatedBy),
INDEX (updatedAt)
);
CREATE TABLE IF NOT EXISTS glossary_term_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
);
-- Set default as false for all existing roles except DataConsumer, to avoid unintended manipulation of roles during migration.
ALTER TABLE role_entity
ADD COLUMN defaultRole BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.defaultRole')),
ADD INDEX(defaultRole);
UPDATE role_entity
SET json = JSON_SET(json, '$.defaultRole', FALSE);
UPDATE role_entity
SET json = JSON_SET(json, '$.defaultRole', TRUE)
WHERE name = 'DataConsumer';
-- Add tag label source
ALTER TABLE tag_usage
ADD COLUMN source TINYINT NOT NULL FIRST, -- Source of tag (either from TagCategory or Glossary)
DROP KEY unique_name,
ADD UNIQUE KEY unique_name(source, tagFQN, targetFQN);
UPDATE tag_usage
SET source = 0
WHERE source IS NULL;

163
bootstrap/sql/mysql/v005__create_db_connection_info.sql
View File

@ -1,163 +0,0 @@
--
-- Drop indexes for deleted boolean column
-- Drop unused indexes for updatedAt and updatedBy
--
ALTER TABLE field_relationship
DROP INDEX edgeIdx;
ALTER TABLE dbservice_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE messaging_service_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE dashboard_service_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE pipeline_service_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE storage_service_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE database_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE table_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE metric_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE report_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE dashboard_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE ml_model_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE pipeline_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE topic_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE chart_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE location_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE thread_entity
DROP INDEX updatedBy;
ALTER TABLE policy_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE team_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE user_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE bot_entity
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE role_entity
DROP INDEX deleted,
DROP INDEX defaultRole,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE tag_category
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE tag
DROP INDEX deleted,
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE airflow_pipeline_entity
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE glossary_entity
DROP INDEX updatedAt,
DROP INDEX updatedBy;
ALTER TABLE glossary_term_entity
DROP INDEX updatedAt,
DROP INDEX updatedBy;
CREATE TABLE IF NOT EXISTS database_schema_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
);
--
-- Drop indexes for deleted boolean column
-- Drop unused indexes for updatedAt and updatedBy
--
RENAME TABLE airflow_pipeline_entity to ingestion_pipeline_entity;
ALTER TABLE tag_category
ADD COLUMN id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL FIRST;
UPDATE tag_category
SET json = JSON_SET(json, '$.id', UUID());
ALTER TABLE tag
ADD COLUMN id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL FIRST;
UPDATE tag
SET json = JSON_SET(json, '$.id', UUID());