yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-17 13:36:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make bootstrap sql files for mysql and postgres consistent with each other (#4541)

Browse Source 
* Make mysql bootstrap file consistent with postgres

* Adding a java code change to trigger all the precommit checks

* Set test container reuse flag to true

* Make Postgres bootstrap files consistent with MySQL

* Test pagination using random jump of limit between 1-5 instead of 1 to reduce test time

* Starting the test container outside @BeforeAll in a static block for container reuse

* Comment printing out list output in pagination tests

* Move starting the test container to @BeforeAll setup method instead of static initialization

* Remove unnecessary tests in PolicyResourceTest to speed up tests

* Add test container reuse for github workflows

* Add test container reuse for github workflows

* Add test container reuse for github workflows

* Change timeouts for the TestContainer

* Add test container reuse for github workflows

* Add test container reuse for github workflows (#4563)

Co-authored-by: Sriharsha Chintalapani <harsha@getcollate.io>
Co-authored-by: Sriharsha Chintalapani <harshach@users.noreply.github.com>
This commit is contained in:
Suresh Srinivas 2022-04-28 07:30:56 -07:00 committed by GitHub
parent c372293188
commit f0139c3ea5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 95 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
bootstrap/sql/com.mysql.cj.jdbc.Driver/v001__create_db_connection_info.sql
View File

@ -9,10 +9,10 @@ CREATE TABLE IF NOT EXISTS entity_relationship (
relation TINYINT NOT NULL,
jsonSchema VARCHAR(256), -- Schema used for generating JSON
json JSON, -- JSON payload with additional information
deleted BOOLEAN NOT NULL DEFAULT 0,
INDEX edgeIdx (fromId, toId, relation),
INDEX fromIdx (fromId, relation),
INDEX toIdx (toId, relation),
deleted BOOLEAN NOT NULL DEFAULT FALSE,
INDEX edge_index (fromId, toId, relation),
INDEX from_index (fromId, relation),
INDEX to_index (toId, relation),
PRIMARY KEY (fromId, toId, relation)
);
@ -29,8 +29,8 @@ CREATE TABLE IF NOT EXISTS field_relationship (
relation TINYINT NOT NULL,
jsonSchema VARCHAR(256), -- Schema used for generating JSON
json JSON, -- JSON payload with additional information
INDEX fromIdx (fromFQN, relation),
INDEX toIdx (toFQN, relation),
INDEX from_index (fromFQN, relation),
INDEX to_index (toFQN, relation),
PRIMARY KEY (fromFQN, toFQN, relation)
);
@ -57,7 +57,7 @@ CREATE TABLE IF NOT EXISTS dbservice_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS messaging_service_entity (
@ -69,7 +69,7 @@ CREATE TABLE IF NOT EXISTS messaging_service_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS dashboard_service_entity (
@ -81,7 +81,7 @@ CREATE TABLE IF NOT EXISTS dashboard_service_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS pipeline_service_entity (
@ -93,7 +93,7 @@ CREATE TABLE IF NOT EXISTS pipeline_service_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS storage_service_entity (
@ -105,7 +105,7 @@ CREATE TABLE IF NOT EXISTS storage_service_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
UNIQUE (name)
);
--
@ -119,7 +119,7 @@ CREATE TABLE IF NOT EXISTS database_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS database_schema_entity (
@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS database_schema_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS table_entity (
@ -141,7 +141,7 @@ CREATE TABLE IF NOT EXISTS table_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS metric_entity (
@ -152,7 +152,7 @@ CREATE TABLE IF NOT EXISTS metric_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS report_entity (
@ -163,7 +163,7 @@ CREATE TABLE IF NOT EXISTS report_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS dashboard_entity (
@ -174,7 +174,7 @@ CREATE TABLE IF NOT EXISTS dashboard_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS ml_model_entity (
@ -185,7 +185,7 @@ CREATE TABLE IF NOT EXISTS ml_model_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS pipeline_entity (
@ -196,7 +196,7 @@ CREATE TABLE IF NOT EXISTS pipeline_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS topic_entity (
@ -207,7 +207,7 @@ CREATE TABLE IF NOT EXISTS topic_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS chart_entity (
@ -218,7 +218,7 @@ CREATE TABLE IF NOT EXISTS chart_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS location_entity (
@ -229,7 +229,7 @@ CREATE TABLE IF NOT EXISTS location_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
--
@ -260,7 +260,7 @@ CREATE TABLE IF NOT EXISTS policy_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
--
@ -275,7 +275,7 @@ CREATE TABLE IF NOT EXISTS ingestion_pipeline_entity (
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
timestamp BIGINT,
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
--
@ -289,7 +289,7 @@ CREATE TABLE IF NOT EXISTS team_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS user_entity (
@ -302,7 +302,7 @@ CREATE TABLE IF NOT EXISTS user_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS bot_entity (
@ -313,35 +313,35 @@ CREATE TABLE IF NOT EXISTS bot_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS role_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
defaultRole BOOLEAN GENERATED ALWAYS AS (json -> '$.defaultRole'),
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
defaultRole BOOLEAN GENERATED ALWAYS AS (json -> '$.defaultRole'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
UNIQUE (name)
);
--
-- Usage table where usage for all the entities is captured
--
CREATE TABLE IF NOT EXISTS entity_usage (
id VARCHAR(36) NOT NULL, -- Unique id of the entity
entityType VARCHAR(20) NOT NULL, -- name of the entity for which this usage is published
usageDate DATE, -- date corresponding to the usage
count1 INT, -- total daily count of use on usageDate
count7 INT, -- rolling count of last 7 days going back from usageDate
count30 INT, -- rolling count of last 30 days going back from usageDate
percentile1 INT, -- percentile rank with in same entity for given usage date
percentile7 INT, -- percentile rank with in same entity for last 7 days of usage
percentile30 INT, -- percentile rank with in same entity for last 30 days of usage
UNIQUE KEY unique_name(usageDate, id)
id VARCHAR(36) NOT NULL, -- Unique id of the entity
entityType VARCHAR(20) NOT NULL, -- name of the entity for which this usage is published
usageDate DATE, -- date corresponding to the usage
count1 INT, -- total daily count of use on usageDate
count7 INT, -- rolling count of last 7 days going back from usageDate
count30 INT, -- rolling count of last 30 days going back from usageDate
percentile1 INT, -- percentile rank with in same entity for given usage date
percentile7 INT, -- percentile rank with in same entity for last 7 days of usage
percentile30 INT, -- percentile rank with in same entity for last 30 days of usage
UNIQUE (usageDate, id)
);
--
@ -354,7 +354,7 @@ CREATE TABLE IF NOT EXISTS tag_category (
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
UNIQUE KEY unique_name(name) -- Unique tag category name
UNIQUE (name) -- Unique tag category name
);
CREATE TABLE IF NOT EXISTS tag (
@ -364,7 +364,7 @@ CREATE TABLE IF NOT EXISTS tag (
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS tag_usage (
@ -373,7 +373,7 @@ CREATE TABLE IF NOT EXISTS tag_usage (
targetFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the entity instance or corresponding field
labelType TINYINT NOT NULL, -- Type of tagging: manual, automated, propagated, derived
state TINYINT NOT NULL, -- State of tagging: suggested or confirmed
UNIQUE KEY unique_name(source, tagFQN, targetFQN)
UNIQUE (source, tagFQN, targetFQN)
);
CREATE TABLE IF NOT EXISTS change_event (
@ -382,9 +382,9 @@ CREATE TABLE IF NOT EXISTS change_event (
userName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.userName') NOT NULL,
eventTime BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
json JSON NOT NULL,
INDEX (eventType),
INDEX (entityType),
INDEX (eventTime)
INDEX event_type_index (eventType),
INDEX entity_type_index (entityType),
INDEX event_time_index (eventTime)
);
CREATE TABLE IF NOT EXISTS webhook_entity (
@ -393,7 +393,7 @@ CREATE TABLE IF NOT EXISTS webhook_entity (
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
json JSON NOT NULL,
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
UNIQUE (name)
-- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
);
@ -405,7 +405,7 @@ CREATE TABLE IF NOT EXISTS glossary_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS glossary_term_entity (
@ -416,6 +416,6 @@ CREATE TABLE IF NOT EXISTS glossary_term_entity (
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName)
UNIQUE (fullyQualifiedName)
);

124
bootstrap/sql/org.postgresql.Driver/v001__create_db_connection_info.sql
View File

@ -8,22 +8,20 @@ $$
-- Table that captures all the relationships between entities
--
CREATE TABLE IF NOT EXISTS entity_relationship (
fromId VARCHAR(36) NOT NULL, -- ID of the from entity
toId VARCHAR(36) NOT NULL, -- ID of the to entity
fromEntity VARCHAR(256) NOT NULL, -- Type name of the from entity
toEntity VARCHAR(256) NOT NULL, -- Type name of to entity
fromId VARCHAR(36) NOT NULL, -- ID of the from entity
toId VARCHAR(36) NOT NULL, -- ID of the to entity
fromEntity VARCHAR(256) NOT NULL, -- Type name of the from entity
toEntity VARCHAR(256) NOT NULL, -- Type name of to entity
relation SMALLINT NOT NULL,
jsonSchema VARCHAR(256), -- Schema used for generating JSON
json JSONB, -- JSON payload with additional information
jsonSchema VARCHAR(256), -- Schema used for generating JSON
json JSONB, -- JSON payload with additional information
deleted BOOLEAN NOT NULL DEFAULT FALSE,
PRIMARY KEY (fromId, toId, relation)
);
CREATE INDEX entity_relationship_edge_index ON entity_relationship(fromId, toId, relation);
CREATE INDEX entity_relationship_from_index ON entity_relationship(fromId, relation);
CREATE INDEX entity_relationship_to_index ON entity_relationship(fromId, toId, relation);
CREATE INDEX entity_relationship_to_index ON entity_relationship(toId, relation);
--
-- Table that captures all the relationships between field of an entity to a field of another entity
@ -31,14 +29,13 @@ CREATE INDEX entity_relationship_to_index ON entity_relationship(fromId, toId, r
-- table2.column8 (toFQN of type table.columns.column)
--
CREATE TABLE IF NOT EXISTS field_relationship (
fromFQN VARCHAR(256) NOT NULL, -- Fully qualified name of entity or field
toFQN VARCHAR(256) NOT NULL, -- Fully qualified name of entity or field
fromType VARCHAR(256) NOT NULL, -- Fully qualified type of entity or field
toType VARCHAR(256) NOT NULL, -- Fully qualified type of entity or field
fromFQN VARCHAR(256) NOT NULL, -- Fully qualified name of entity or field
toFQN VARCHAR(256) NOT NULL, -- Fully qualified name of entity or field
fromType VARCHAR(256) NOT NULL, -- Fully qualified type of entity or field
toType VARCHAR(256) NOT NULL, -- Fully qualified type of entity or field
relation SMALLINT NOT NULL,
jsonSchema VARCHAR(256), -- Schema used for generating JSON
json JSONB, -- JSON payload with additional information
jsonSchema VARCHAR(256), -- Schema used for generating JSON
json JSONB, -- JSON payload with additional information
PRIMARY KEY (fromFQN, toFQN, relation)
);
@ -49,11 +46,10 @@ CREATE INDEX field_relationship_to_index ON field_relationship(toFQN, relation);
-- Used for storing additional metadata for an entity
--
CREATE TABLE IF NOT EXISTS entity_extension (
id VARCHAR(36) NOT NULL, -- ID of the from entity
extension VARCHAR(256) NOT NULL, -- Extension name same as entity.fieldName
jsonSchema VARCHAR(256) NOT NULL, -- Schema used for generating JSON
id VARCHAR(36) NOT NULL, -- ID of the from entity
extension VARCHAR(256) NOT NULL, -- Extension name same as entity.fieldName
jsonSchema VARCHAR(256) NOT NULL, -- Schema used for generating JSON
json JSONB NOT NULL,
PRIMARY KEY (id, extension)
);
@ -65,11 +61,9 @@ CREATE TABLE IF NOT EXISTS dbservice_entity (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (name)
);
@ -79,11 +73,9 @@ CREATE TABLE IF NOT EXISTS messaging_service_entity (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (name)
);
@ -93,11 +85,9 @@ CREATE TABLE IF NOT EXISTS dashboard_service_entity (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (name)
);
@ -107,11 +97,9 @@ CREATE TABLE IF NOT EXISTS pipeline_service_entity (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (name)
);
@ -121,11 +109,9 @@ CREATE TABLE IF NOT EXISTS storage_service_entity (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (name)
);
@ -137,11 +123,9 @@ CREATE TABLE IF NOT EXISTS database_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
@ -150,11 +134,9 @@ CREATE TABLE IF NOT EXISTS database_schema_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt') :: bigint) STORED NOT NULL NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted') :: boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
@ -163,11 +145,9 @@ CREATE TABLE IF NOT EXISTS table_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
@ -176,11 +156,9 @@ CREATE TABLE IF NOT EXISTS metric_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
@ -189,11 +167,9 @@ CREATE TABLE IF NOT EXISTS report_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
@ -202,11 +178,9 @@ CREATE TABLE IF NOT EXISTS dashboard_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
@ -215,11 +189,9 @@ CREATE TABLE IF NOT EXISTS ml_model_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
@ -228,11 +200,9 @@ CREATE TABLE IF NOT EXISTS pipeline_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
@ -241,11 +211,9 @@ CREATE TABLE IF NOT EXISTS topic_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
@ -254,11 +222,9 @@ CREATE TABLE IF NOT EXISTS chart_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
@ -267,11 +233,9 @@ CREATE TABLE IF NOT EXISTS location_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
@ -285,13 +249,11 @@ CREATE TABLE IF NOT EXISTS thread_entity (
entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> 'about') STORED NOT NULL,
assignedTo VARCHAR(256) GENERATED ALWAYS AS (json ->> 'addressedTo') STORED,
json JSONB NOT NULL,
createdAt BIGINT GENERATED ALWAYS AS ((json ->> 'threadTs')::bigint) STORED NOT NULL,
createdBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'createdBy') STORED NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
resolved BOOLEAN GENERATED ALWAYS AS ((json ->> 'resolved')::boolean) STORED,
PRIMARY KEY (id)
);
@ -302,11 +264,9 @@ CREATE TABLE IF NOT EXISTS policy_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
@ -315,17 +275,15 @@ CREATE TABLE IF NOT EXISTS policy_entity (
-- Ingestion related tables
--
CREATE TABLE IF NOT EXISTS ingestion_pipeline_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
timestamp BIGINT,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
timestamp BIGINT,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
--
@ -335,11 +293,9 @@ CREATE TABLE IF NOT EXISTS team_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (name)
);
@ -350,11 +306,9 @@ CREATE TABLE IF NOT EXISTS user_entity (
email VARCHAR(256) GENERATED ALWAYS AS (json ->> 'email') STORED NOT NULL,
deactivated VARCHAR(8) GENERATED ALWAYS AS (json ->> 'deactivated') STORED,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (name)
);
@ -363,11 +317,9 @@ CREATE TABLE IF NOT EXISTS bot_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (name)
);
@ -377,11 +329,9 @@ CREATE TABLE IF NOT EXISTS role_entity (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
defaultRole BOOLEAN GENERATED ALWAYS AS ((json ->> 'defaultRole')::boolean) STORED,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (name)
);
@ -406,36 +356,31 @@ CREATE TABLE IF NOT EXISTS entity_usage (
-- Tag related tables
--
CREATE TABLE IF NOT EXISTS tag_category (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
json JSONB NOT NULL, -- JSON stores category information and does not store children
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
UNIQUE (name) -- Unique tag category name
);
CREATE TABLE IF NOT EXISTS tag (
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL, -- JSON stores all tag attributes and does not store children
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
UNIQUE (fullyQualifiedName)
);
CREATE TABLE IF NOT EXISTS tag_usage (
tagFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the tag
targetFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the entity instance or corresponding field
labelType SMALLINT, -- Type of tagging: manual, automated, propagated NOT NULL, derived
state SMALLINT NOT NULL, -- State of tagging: suggested or confirmed
source SMALLINT NOT NULL, -- Source of tag (either from TagCategory or Glossary)
source SMALLINT NOT NULL, -- Source of the tag label
tagFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the tag
targetFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the entity instance or corresponding field
labelType SMALLINT NOT NULL, -- Type of tagging: manual, automated, propagated, derived
state SMALLINT NOT NULL, -- State of tagging: suggested or confirmed
UNIQUE (source, tagFQN, targetFQN)
);
@ -456,7 +401,6 @@ CREATE TABLE IF NOT EXISTS webhook_entity (
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
json JSON NOT NULL,
PRIMARY KEY (id),
UNIQUE (name)
-- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
@ -466,11 +410,9 @@ CREATE TABLE IF NOT EXISTS glossary_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (name)
);
@ -479,11 +421,9 @@ CREATE TABLE IF NOT EXISTS glossary_term_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java
View File

@ -52,9 +52,6 @@ public final class Entity {
// Canonical entity name to corresponding EntityRepository map
private static final Map<String, EntityRepository<?>> ENTITY_REPOSITORY_MAP = new HashMap<>();
// Entity class to entity repository map
private static final Map<Class<?>, EntityRepository<?>> CLASS_ENTITY_REPOSITORY_MAP = new HashMap<>();
// Common field names
public static final String FIELD_OWNER = "owner";
public static final String FIELD_NAME = "name";
@ -140,7 +137,6 @@ public final class Entity {
DAO_MAP.put(entity, dao);
ENTITY_REPOSITORY_MAP.put(entity, entityRepository);
CANONICAL_ENTITY_NAME_MAP.put(entity.toLowerCase(Locale.ROOT), entity);
CLASS_ENTITY_REPOSITORY_MAP.put(clazz, entityRepository);
LOG.info(
"Registering entity {} {} {} {}",
clazz,

9
catalog-rest-service/src/test/java/org/openmetadata/catalog/CatalogApplicationTest.java
View File

@ -41,7 +41,6 @@ public abstract class CatalogApplicationTest {
static {
CollectionRegistry.addTestResource(webhookCallbackResource);
Fernet.getInstance().setFernetKey(FERNET_KEY_1);
}
@ -54,6 +53,9 @@ public abstract class CatalogApplicationTest {
SQL_CONTAINER =
(JdbcDatabaseContainer<?>)
Class.forName(jdbcContainerClassName).getConstructor(String.class).newInstance(jdbcContainerImage);
SQL_CONTAINER.withReuse(true);
SQL_CONTAINER.withStartupTimeoutSeconds(240);
SQL_CONTAINER.withConnectTimeoutSeconds(240);
SQL_CONTAINER.start();
final String migrationScripsLocation =
@ -84,11 +86,8 @@ public abstract class CatalogApplicationTest {
@AfterAll
public static void stopApplication() {
// If beforeAll causes and exception AfterAll still gets called before that exception is thrown.
// If BeforeAll causes and exception AfterAll still gets called before that exception is thrown.
// If a NullPointerException is thrown during the cleanup of above it will eat the initial error
if (SQL_CONTAINER != null) {
SQL_CONTAINER.stop();
}
if (APP != null) {
APP.after();
}

13
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java
View File

@ -393,6 +393,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
};
// Test listing entities that include deleted, non-deleted, and all the entities
Random random = new Random();
for (String include : List.of("non-deleted", "all", "deleted")) {
if (!supportsSoftDelete && include.equals("deleted")) {
continue;
@ -405,9 +406,9 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
int totalRecords = allEntities.getData().size();
printEntities(allEntities);
// List entity with "limit" set from 1 to maxTables size
// List entity with "limit" set from 1 to maxTables size with random jumps (to reduce the test time)
// Each time compare the returned list with allTables list to make sure right results are returned
for (int limit = 1; limit < maxEntities; limit++) {
for (int limit = 1; limit < maxEntities; limit += random.nextInt(5) + 1) {
String after = null;
String before;
int pageCount = 0;
@ -416,7 +417,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
ResultList<T> backwardPage;
boolean foundDeleted = false;
do { // For each limit (or page size) - forward scroll till the end
LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after);
LOG.debug("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after);
forwardPage = listEntities(queryParams, limit, null, after, ADMIN_AUTH_HEADERS);
foundDeleted = forwardPage.getData().stream().anyMatch(matchDeleted) || foundDeleted;
after = forwardPage.getPaging().getAfter();
@ -447,7 +448,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
indexInAllTables = totalRecords - limit - forwardPage.getData().size();
foundDeleted = false;
do {
LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before);
LOG.debug("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before);
forwardPage = listEntities(queryParams, limit, before, null, ADMIN_AUTH_HEADERS);
foundDeleted = forwardPage.getData().stream().anyMatch(matchDeleted) || foundDeleted;
printEntities(forwardPage);
@ -1819,8 +1820,8 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
}
private void printEntities(ResultList<T> list) {
list.getData().forEach(e -> LOG.info("{} {}", entityClass, getEntityInterface(e).getFullyQualifiedName()));
LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
list.getData().forEach(e -> LOG.debug("{} {}", entityClass, getEntityInterface(e).getFullyQualifiedName()));
LOG.debug("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
}
public void assertEntityDeleted(EntityInterface<T> entityInterface, boolean hardDelete) {

120
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java
View File

@ -15,11 +15,9 @@ package org.openmetadata.catalog.resources.policies;
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.fail;
import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.assertEntityPagination;
import static org.openmetadata.catalog.util.TestUtils.assertListNotNull;
import static org.openmetadata.catalog.util.TestUtils.assertListNull;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
@ -31,9 +29,7 @@ import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.ws.rs.client.WebTarget;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.BeforeAll;
@ -178,90 +174,6 @@ public class PolicyResourceTest extends EntityResourceTest<Policy, CreatePolicy>
getEntityName(test)));
}
@Test
void get_PolicyListWithInvalidLimitOffset_4xx() {
// Limit must be >= 1 and <= 1000,000
assertResponse(
() -> listPolicies(null, -1, null, null, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"[query param limit must be greater than or equal to 0]");
assertResponse(
() -> listPolicies(null, 1000001, null, null, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"[query param limit must be less than or equal to 1000000]");
}
@Test
void get_PolicyListWithInvalidPaginationCursors_4xx() {
// Passing both before and after cursors is invalid
assertResponse(
() -> listPolicies(null, 1, "", "", ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"Only one of before or after query parameter allowed");
}
@Test
void get_PolicyListWithValidLimitOffset_4xx(TestInfo test) throws HttpResponseException {
// Create a large number of Policies
int maxPolicies = 40;
for (int i = 0; i < maxPolicies; i++) {
createEntity(createRequest(test, i), ADMIN_AUTH_HEADERS);
}
// List all Policies
PolicyList allPolicies = listPolicies(null, 1000000, null, null, ADMIN_AUTH_HEADERS);
int totalRecords = allPolicies.getData().size();
printPolicies(allPolicies);
// List limit number Policies at a time at various offsets and ensure right results are returned
for (int limit = 1; limit < maxPolicies; limit++) {
String after = null;
String before;
int pageCount = 0;
int indexInAllPolicies = 0;
PolicyList forwardPage;
PolicyList backwardPage;
do { // For each limit (or page size) - forward scroll till the end
LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after);
forwardPage = listPolicies(null, limit, null, after, ADMIN_AUTH_HEADERS);
printPolicies(forwardPage);
after = forwardPage.getPaging().getAfter();
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allPolicies.getData(), forwardPage, limit, indexInAllPolicies);
if (pageCount == 0) { // CASE 0 - First page is being returned. Therefore, before cursor is null
assertNull(before);
} else {
// Make sure scrolling back based on before cursor returns the correct result
backwardPage = listPolicies(null, limit, before, null, ADMIN_AUTH_HEADERS);
assertEntityPagination(allPolicies.getData(), backwardPage, limit, (indexInAllPolicies - limit));
}
indexInAllPolicies += forwardPage.getData().size();
pageCount++;
} while (after != null);
// We have now reached the last page - test backward scroll till the beginning
pageCount = 0;
indexInAllPolicies = totalRecords - limit - forwardPage.getData().size();
do {
LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before);
forwardPage = listPolicies(null, limit, before, null, ADMIN_AUTH_HEADERS);
printPolicies(forwardPage);
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allPolicies.getData(), forwardPage, limit, indexInAllPolicies);
pageCount++;
indexInAllPolicies -= forwardPage.getData().size();
} while (before != null);
}
}
private void printPolicies(PolicyList list) {
list.getData().forEach(Policy -> LOG.info("DB {}", Policy.getFullyQualifiedName()));
LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
}
@Test
void patch_PolicyAttributes_200_ok(TestInfo test) throws IOException {
Policy policy = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS).withLocation(null);
@ -305,46 +217,22 @@ public class PolicyResourceTest extends EntityResourceTest<Policy, CreatePolicy>
String fields = "";
policy =
byName
? getPolicyByName(policy.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS)
: getPolicy(policy.getId(), fields, ADMIN_AUTH_HEADERS);
? getEntityByName(policy.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS)
: getEntity(policy.getId(), fields, ADMIN_AUTH_HEADERS);
assertListNull(policy.getOwner(), policy.getLocation());
// .../policies?fields=owner,displayName,policyUrl
fields = "owner,location";
policy =
byName
? getPolicyByName(policy.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS)
: getPolicy(policy.getId(), fields, ADMIN_AUTH_HEADERS);
? getEntityByName(policy.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS)
: getEntity(policy.getId(), fields, ADMIN_AUTH_HEADERS);
// Field location is set during creation - tested elsewhere
assertListNotNull(policy.getOwner() /*, policy.getLocation()*/);
// Checks for other owner, tags, and followers is done in the base class
return getEntityInterface(policy);
}
public static Policy getPolicy(UUID id, String fields, Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = getResource("policies/" + id);
target = fields != null ? target.queryParam("fields", fields) : target;
return TestUtils.get(target, Policy.class, authHeaders);
}
public static Policy getPolicyByName(String fqn, String fields, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = getResource("policies/name/" + fqn);
target = fields != null ? target.queryParam("fields", fields) : target;
return TestUtils.get(target, Policy.class, authHeaders);
}
public static PolicyList listPolicies(
String fields, Integer limitParam, String before, String after, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = getResource("policies");
target = fields != null ? target.queryParam("fields", fields) : target;
target = limitParam != null ? target.queryParam("limit", limitParam) : target;
target = before != null ? target.queryParam("before", before) : target;
target = after != null ? target.queryParam("after", after) : target;
return TestUtils.get(target, PolicyList.class, authHeaders);
}
private CreatePolicy createAccessControlPolicyWithRules(String name, List<Rule> rules) {
return new CreatePolicy()
.withName(name)