mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-07-05 08:05:28 +00:00
22b0f44e38
* Update mapping * Prep * prep * Prep query migration * Add query index deletion fix * Docs and Maven CI fix * Fix tests * Add service filter * Add query entity FQN col migration * Fix lint * supported serviceFQN in query api * Prep repo * Prep ES query search * Do not recompute lineage * Format * Fix test --------- Co-authored-by: Ashish Gupta <ashish@getcollate.io>
107 lines
4.5 KiB
SQL
107 lines
4.5 KiB
SQL
-- column deleted not needed for entities that don't support soft delete
|
|
ALTER TABLE query_entity DROP COLUMN deleted;
|
|
ALTER TABLE event_subscription_entity DROP COLUMN deleted;
|
|
|
|
-- create domain entity table
|
|
CREATE TABLE IF NOT EXISTS domain_entity (
|
|
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
|
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
|
fqnHash VARCHAR(256) NOT NULL,
|
|
json JSONB NOT NULL,
|
|
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
|
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
|
PRIMARY KEY (id),
|
|
UNIQUE (fqnHash)
|
|
);
|
|
|
|
-- create data product entity table
|
|
CREATE TABLE IF NOT EXISTS data_product_entity (
|
|
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
|
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
|
fqnHash VARCHAR(256) NOT NULL,
|
|
json JSONB NOT NULL,
|
|
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
|
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
|
PRIMARY KEY (id),
|
|
UNIQUE (fqnHash)
|
|
);
|
|
|
|
-- create search service entity
|
|
CREATE TABLE IF NOT EXISTS search_service_entity (
|
|
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
|
nameHash VARCHAR(256) NOT NULL,
|
|
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
|
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
|
|
json JSONB NOT NULL,
|
|
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
|
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
|
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
|
PRIMARY KEY (id),
|
|
UNIQUE (nameHash)
|
|
);
|
|
|
|
-- create search index entity
|
|
CREATE TABLE IF NOT EXISTS search_index_entity (
|
|
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
|
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
|
fqnHash VARCHAR(256) NOT NULL,
|
|
json JSONB NOT NULL,
|
|
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
|
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
|
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
|
PRIMARY KEY (id),
|
|
UNIQUE (fqnHash)
|
|
);
|
|
|
|
-- We were hardcoding retries to 0. Since we are now using the IngestionPipeline to set them, keep existing ones to 0.
|
|
UPDATE ingestion_pipeline_entity
|
|
SET json = jsonb_set(json::jsonb, '{airflowConfig,retries}', '0', true);
|
|
|
|
-- create stored procedure entity
|
|
CREATE TABLE IF NOT EXISTS stored_procedure_entity (
|
|
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
|
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
|
fqnHash VARCHAR(256) NOT NULL,
|
|
json JSONB NOT NULL,
|
|
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
|
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
|
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
|
PRIMARY KEY (id),
|
|
UNIQUE (fqnHash)
|
|
);
|
|
|
|
-- create entity extension table for table entity
|
|
CREATE TABLE IF NOT EXISTS table_entity_extension (
|
|
id VARCHAR(36) NOT NULL, -- ID of the from entity
|
|
extension VARCHAR(256) NOT NULL, -- Extension name same as entity.fieldName
|
|
jsonSchema VARCHAR(256) NOT NULL, -- Schema used for generating JSON
|
|
json JSONB NOT NULL,
|
|
PRIMARY KEY (id, extension)
|
|
);
|
|
|
|
-- Add index on fromId and fromEntity columns
|
|
CREATE INDEX from_entity_type_index ON entity_relationship (fromId, fromEntity);
|
|
|
|
-- Add index on toId and toEntity columns
|
|
CREATE INDEX to_entity_type_index ON entity_relationship (toId, toEntity);
|
|
|
|
ALTER TABLE tag DROP CONSTRAINT IF EXISTS tag_fqnhash_key;
|
|
|
|
ALTER TABLE tag ADD CONSTRAINT unique_fqnHash UNIQUE (fqnHash);
|
|
|
|
ALTER TABLE tag ADD CONSTRAINT tag_pk PRIMARY KEY (id);
|
|
|
|
|
|
-- rename viewParsingTimeoutLimit for queryParsingTimeoutLimit
|
|
UPDATE ingestion_pipeline_entity
|
|
SET json = jsonb_set(
|
|
json::jsonb #- '{sourceConfig,config,viewParsingTimeoutLimit}',
|
|
'{sourceConfig,config,queryParsingTimeoutLimit}',
|
|
(json #> '{sourceConfig,config,viewParsingTimeoutLimit}')::jsonb,
|
|
true
|
|
)
|
|
WHERE json #>> '{pipelineType}' = 'metadata';
|
|
|
|
-- Query Entity supports service, which requires FQN for name
|
|
ALTER TABLE query_entity RENAME COLUMN nameHash TO fqnHash;
|