From f0139c3ea59f219c8c6c9192e1f69b66e09b2ff8 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 28 Apr 2022 07:30:56 -0700 Subject: [PATCH] Make bootstrap sql files for mysql and postgres consistent with each other (#4541) * Make mysql bootstrap file consistent with postgres * Adding a java code change to trigger all the precommit checks * Set test container reuse flag to true * Make Postgres bootstrap files consistent with MySQL * Test pagination using random jump of limit between 1-5 instead of 1 to reduce test time * Starting the test container outside @BeforeAll in a static block for container reuse * Comment printing out list output in pagination tests * Move starting the test container to @BeforeAll setup method instead of static initialization * Remove unnecessary tests in PolicyResourceTest to speed up tests * Add test container reuse for github workflows * Add test container reuse for github workflows * Add test container reuse for github workflows * Change timeouts for the TestContainer * Add test container reuse for github workflows * Add test container reuse for github workflows (#4563) Co-authored-by: Sriharsha Chintalapani Co-authored-by: Sriharsha Chintalapani --- .../v001__create_db_connection_info.sql | 96 +++++++------- .../v001__create_db_connection_info.sql | 124 +++++------------- .../java/org/openmetadata/catalog/Entity.java | 4 - .../catalog/CatalogApplicationTest.java | 9 +- .../catalog/resources/EntityResourceTest.java | 13 +- .../policies/PolicyResourceTest.java | 120 +---------------- 6 files changed, 95 insertions(+), 271 deletions(-) diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v001__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v001__create_db_connection_info.sql index 22c3197617f..6664300122b 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v001__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v001__create_db_connection_info.sql @@ -9,10 +9,10 @@ CREATE TABLE IF NOT EXISTS entity_relationship ( relation TINYINT NOT NULL, jsonSchema VARCHAR(256), -- Schema used for generating JSON json JSON, -- JSON payload with additional information - deleted BOOLEAN NOT NULL DEFAULT 0, - INDEX edgeIdx (fromId, toId, relation), - INDEX fromIdx (fromId, relation), - INDEX toIdx (toId, relation), + deleted BOOLEAN NOT NULL DEFAULT FALSE, + INDEX edge_index (fromId, toId, relation), + INDEX from_index (fromId, relation), + INDEX to_index (toId, relation), PRIMARY KEY (fromId, toId, relation) ); @@ -29,8 +29,8 @@ CREATE TABLE IF NOT EXISTS field_relationship ( relation TINYINT NOT NULL, jsonSchema VARCHAR(256), -- Schema used for generating JSON json JSON, -- JSON payload with additional information - INDEX fromIdx (fromFQN, relation), - INDEX toIdx (toFQN, relation), + INDEX from_index (fromFQN, relation), + INDEX to_index (toFQN, relation), PRIMARY KEY (fromFQN, toFQN, relation) ); @@ -57,7 +57,7 @@ CREATE TABLE IF NOT EXISTS dbservice_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(name) + UNIQUE (name) ); CREATE TABLE IF NOT EXISTS messaging_service_entity ( @@ -69,7 +69,7 @@ CREATE TABLE IF NOT EXISTS messaging_service_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(name) + UNIQUE (name) ); CREATE TABLE IF NOT EXISTS dashboard_service_entity ( @@ -81,7 +81,7 @@ CREATE TABLE IF NOT EXISTS dashboard_service_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(name) + UNIQUE (name) ); CREATE TABLE IF NOT EXISTS pipeline_service_entity ( @@ -93,7 +93,7 @@ CREATE TABLE IF NOT EXISTS pipeline_service_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(name) + UNIQUE (name) ); CREATE TABLE IF NOT EXISTS storage_service_entity ( @@ -105,7 +105,7 @@ CREATE TABLE IF NOT EXISTS storage_service_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(name) + UNIQUE (name) ); -- @@ -119,7 +119,7 @@ CREATE TABLE IF NOT EXISTS database_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); CREATE TABLE IF NOT EXISTS database_schema_entity ( @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS database_schema_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); CREATE TABLE IF NOT EXISTS table_entity ( @@ -141,7 +141,7 @@ CREATE TABLE IF NOT EXISTS table_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); CREATE TABLE IF NOT EXISTS metric_entity ( @@ -152,7 +152,7 @@ CREATE TABLE IF NOT EXISTS metric_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); CREATE TABLE IF NOT EXISTS report_entity ( @@ -163,7 +163,7 @@ CREATE TABLE IF NOT EXISTS report_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); CREATE TABLE IF NOT EXISTS dashboard_entity ( @@ -174,7 +174,7 @@ CREATE TABLE IF NOT EXISTS dashboard_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); CREATE TABLE IF NOT EXISTS ml_model_entity ( @@ -185,7 +185,7 @@ CREATE TABLE IF NOT EXISTS ml_model_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); CREATE TABLE IF NOT EXISTS pipeline_entity ( @@ -196,7 +196,7 @@ CREATE TABLE IF NOT EXISTS pipeline_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); CREATE TABLE IF NOT EXISTS topic_entity ( @@ -207,7 +207,7 @@ CREATE TABLE IF NOT EXISTS topic_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); CREATE TABLE IF NOT EXISTS chart_entity ( @@ -218,7 +218,7 @@ CREATE TABLE IF NOT EXISTS chart_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); CREATE TABLE IF NOT EXISTS location_entity ( @@ -229,7 +229,7 @@ CREATE TABLE IF NOT EXISTS location_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); -- @@ -260,7 +260,7 @@ CREATE TABLE IF NOT EXISTS policy_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); -- @@ -275,7 +275,7 @@ CREATE TABLE IF NOT EXISTS ingestion_pipeline_entity ( deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), timestamp BIGINT, PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); -- @@ -289,7 +289,7 @@ CREATE TABLE IF NOT EXISTS team_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(name) + UNIQUE (name) ); CREATE TABLE IF NOT EXISTS user_entity ( @@ -302,7 +302,7 @@ CREATE TABLE IF NOT EXISTS user_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(name) + UNIQUE (name) ); CREATE TABLE IF NOT EXISTS bot_entity ( @@ -313,35 +313,35 @@ CREATE TABLE IF NOT EXISTS bot_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(name) + UNIQUE (name) ); CREATE TABLE IF NOT EXISTS role_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, + defaultRole BOOLEAN GENERATED ALWAYS AS (json -> '$.defaultRole'), json JSON NOT NULL, updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), - defaultRole BOOLEAN GENERATED ALWAYS AS (json -> '$.defaultRole'), PRIMARY KEY (id), - UNIQUE KEY unique_name(name) + UNIQUE (name) ); -- -- Usage table where usage for all the entities is captured -- CREATE TABLE IF NOT EXISTS entity_usage ( - id VARCHAR(36) NOT NULL, -- Unique id of the entity - entityType VARCHAR(20) NOT NULL, -- name of the entity for which this usage is published - usageDate DATE, -- date corresponding to the usage - count1 INT, -- total daily count of use on usageDate - count7 INT, -- rolling count of last 7 days going back from usageDate - count30 INT, -- rolling count of last 30 days going back from usageDate - percentile1 INT, -- percentile rank with in same entity for given usage date - percentile7 INT, -- percentile rank with in same entity for last 7 days of usage - percentile30 INT, -- percentile rank with in same entity for last 30 days of usage - UNIQUE KEY unique_name(usageDate, id) + id VARCHAR(36) NOT NULL, -- Unique id of the entity + entityType VARCHAR(20) NOT NULL, -- name of the entity for which this usage is published + usageDate DATE, -- date corresponding to the usage + count1 INT, -- total daily count of use on usageDate + count7 INT, -- rolling count of last 7 days going back from usageDate + count30 INT, -- rolling count of last 30 days going back from usageDate + percentile1 INT, -- percentile rank with in same entity for given usage date + percentile7 INT, -- percentile rank with in same entity for last 7 days of usage + percentile30 INT, -- percentile rank with in same entity for last 30 days of usage + UNIQUE (usageDate, id) ); -- @@ -354,7 +354,7 @@ CREATE TABLE IF NOT EXISTS tag_category ( updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), - UNIQUE KEY unique_name(name) -- Unique tag category name + UNIQUE (name) -- Unique tag category name ); CREATE TABLE IF NOT EXISTS tag ( @@ -364,7 +364,7 @@ CREATE TABLE IF NOT EXISTS tag ( updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); CREATE TABLE IF NOT EXISTS tag_usage ( @@ -373,7 +373,7 @@ CREATE TABLE IF NOT EXISTS tag_usage ( targetFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the entity instance or corresponding field labelType TINYINT NOT NULL, -- Type of tagging: manual, automated, propagated, derived state TINYINT NOT NULL, -- State of tagging: suggested or confirmed - UNIQUE KEY unique_name(source, tagFQN, targetFQN) + UNIQUE (source, tagFQN, targetFQN) ); CREATE TABLE IF NOT EXISTS change_event ( @@ -382,9 +382,9 @@ CREATE TABLE IF NOT EXISTS change_event ( userName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.userName') NOT NULL, eventTime BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL, json JSON NOT NULL, - INDEX (eventType), - INDEX (entityType), - INDEX (eventTime) + INDEX event_type_index (eventType), + INDEX entity_type_index (entityType), + INDEX event_time_index (eventTime) ); CREATE TABLE IF NOT EXISTS webhook_entity ( @@ -393,7 +393,7 @@ CREATE TABLE IF NOT EXISTS webhook_entity ( deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), json JSON NOT NULL, PRIMARY KEY (id), - UNIQUE KEY unique_name(name) + UNIQUE (name) -- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook ); @@ -405,7 +405,7 @@ CREATE TABLE IF NOT EXISTS glossary_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(name) + UNIQUE (name) ); CREATE TABLE IF NOT EXISTS glossary_term_entity ( @@ -416,6 +416,6 @@ CREATE TABLE IF NOT EXISTS glossary_term_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE KEY unique_name(fullyQualifiedName) + UNIQUE (fullyQualifiedName) ); diff --git a/bootstrap/sql/org.postgresql.Driver/v001__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v001__create_db_connection_info.sql index 3d28d41a077..e6b6b6e22b5 100644 --- a/bootstrap/sql/org.postgresql.Driver/v001__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v001__create_db_connection_info.sql @@ -8,22 +8,20 @@ $$ -- Table that captures all the relationships between entities -- CREATE TABLE IF NOT EXISTS entity_relationship ( - fromId VARCHAR(36) NOT NULL, -- ID of the from entity - toId VARCHAR(36) NOT NULL, -- ID of the to entity - fromEntity VARCHAR(256) NOT NULL, -- Type name of the from entity - toEntity VARCHAR(256) NOT NULL, -- Type name of to entity + fromId VARCHAR(36) NOT NULL, -- ID of the from entity + toId VARCHAR(36) NOT NULL, -- ID of the to entity + fromEntity VARCHAR(256) NOT NULL, -- Type name of the from entity + toEntity VARCHAR(256) NOT NULL, -- Type name of to entity relation SMALLINT NOT NULL, - jsonSchema VARCHAR(256), -- Schema used for generating JSON - json JSONB, -- JSON payload with additional information - + jsonSchema VARCHAR(256), -- Schema used for generating JSON + json JSONB, -- JSON payload with additional information deleted BOOLEAN NOT NULL DEFAULT FALSE, - PRIMARY KEY (fromId, toId, relation) ); CREATE INDEX entity_relationship_edge_index ON entity_relationship(fromId, toId, relation); CREATE INDEX entity_relationship_from_index ON entity_relationship(fromId, relation); -CREATE INDEX entity_relationship_to_index ON entity_relationship(fromId, toId, relation); +CREATE INDEX entity_relationship_to_index ON entity_relationship(toId, relation); -- -- Table that captures all the relationships between field of an entity to a field of another entity @@ -31,14 +29,13 @@ CREATE INDEX entity_relationship_to_index ON entity_relationship(fromId, toId, r -- table2.column8 (toFQN of type table.columns.column) -- CREATE TABLE IF NOT EXISTS field_relationship ( - fromFQN VARCHAR(256) NOT NULL, -- Fully qualified name of entity or field - toFQN VARCHAR(256) NOT NULL, -- Fully qualified name of entity or field - fromType VARCHAR(256) NOT NULL, -- Fully qualified type of entity or field - toType VARCHAR(256) NOT NULL, -- Fully qualified type of entity or field + fromFQN VARCHAR(256) NOT NULL, -- Fully qualified name of entity or field + toFQN VARCHAR(256) NOT NULL, -- Fully qualified name of entity or field + fromType VARCHAR(256) NOT NULL, -- Fully qualified type of entity or field + toType VARCHAR(256) NOT NULL, -- Fully qualified type of entity or field relation SMALLINT NOT NULL, - jsonSchema VARCHAR(256), -- Schema used for generating JSON - json JSONB, -- JSON payload with additional information - + jsonSchema VARCHAR(256), -- Schema used for generating JSON + json JSONB, -- JSON payload with additional information PRIMARY KEY (fromFQN, toFQN, relation) ); @@ -49,11 +46,10 @@ CREATE INDEX field_relationship_to_index ON field_relationship(toFQN, relation); -- Used for storing additional metadata for an entity -- CREATE TABLE IF NOT EXISTS entity_extension ( - id VARCHAR(36) NOT NULL, -- ID of the from entity - extension VARCHAR(256) NOT NULL, -- Extension name same as entity.fieldName - jsonSchema VARCHAR(256) NOT NULL, -- Schema used for generating JSON + id VARCHAR(36) NOT NULL, -- ID of the from entity + extension VARCHAR(256) NOT NULL, -- Extension name same as entity.fieldName + jsonSchema VARCHAR(256) NOT NULL, -- Schema used for generating JSON json JSONB NOT NULL, - PRIMARY KEY (id, extension) ); @@ -65,11 +61,9 @@ CREATE TABLE IF NOT EXISTS dbservice_entity ( name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (name) ); @@ -79,11 +73,9 @@ CREATE TABLE IF NOT EXISTS messaging_service_entity ( name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (name) ); @@ -93,11 +85,9 @@ CREATE TABLE IF NOT EXISTS dashboard_service_entity ( name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (name) ); @@ -107,11 +97,9 @@ CREATE TABLE IF NOT EXISTS pipeline_service_entity ( name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (name) ); @@ -121,11 +109,9 @@ CREATE TABLE IF NOT EXISTS storage_service_entity ( name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (name) ); @@ -137,11 +123,9 @@ CREATE TABLE IF NOT EXISTS database_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); @@ -150,11 +134,9 @@ CREATE TABLE IF NOT EXISTS database_schema_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt') :: bigint) STORED NOT NULL NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted') :: boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); @@ -163,11 +145,9 @@ CREATE TABLE IF NOT EXISTS table_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); @@ -176,11 +156,9 @@ CREATE TABLE IF NOT EXISTS metric_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); @@ -189,11 +167,9 @@ CREATE TABLE IF NOT EXISTS report_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); @@ -202,11 +178,9 @@ CREATE TABLE IF NOT EXISTS dashboard_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); @@ -215,11 +189,9 @@ CREATE TABLE IF NOT EXISTS ml_model_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); @@ -228,11 +200,9 @@ CREATE TABLE IF NOT EXISTS pipeline_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); @@ -241,11 +211,9 @@ CREATE TABLE IF NOT EXISTS topic_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); @@ -254,11 +222,9 @@ CREATE TABLE IF NOT EXISTS chart_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); @@ -267,11 +233,9 @@ CREATE TABLE IF NOT EXISTS location_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); @@ -285,13 +249,11 @@ CREATE TABLE IF NOT EXISTS thread_entity ( entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> 'about') STORED NOT NULL, assignedTo VARCHAR(256) GENERATED ALWAYS AS (json ->> 'addressedTo') STORED, json JSONB NOT NULL, - createdAt BIGINT GENERATED ALWAYS AS ((json ->> 'threadTs')::bigint) STORED NOT NULL, createdBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'createdBy') STORED NOT NULL, updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, resolved BOOLEAN GENERATED ALWAYS AS ((json ->> 'resolved')::boolean) STORED, - PRIMARY KEY (id) ); @@ -302,11 +264,9 @@ CREATE TABLE IF NOT EXISTS policy_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); @@ -315,17 +275,15 @@ CREATE TABLE IF NOT EXISTS policy_entity ( -- Ingestion related tables -- CREATE TABLE IF NOT EXISTS ingestion_pipeline_entity ( - id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, - fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, - json JSON NOT NULL, - - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL, - updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, - deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - timestamp BIGINT, - - PRIMARY KEY (id), - UNIQUE (fullyQualifiedName) + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, + timestamp BIGINT, + PRIMARY KEY (id), + UNIQUE (fullyQualifiedName) ); -- @@ -335,11 +293,9 @@ CREATE TABLE IF NOT EXISTS team_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (name) ); @@ -350,11 +306,9 @@ CREATE TABLE IF NOT EXISTS user_entity ( email VARCHAR(256) GENERATED ALWAYS AS (json ->> 'email') STORED NOT NULL, deactivated VARCHAR(8) GENERATED ALWAYS AS (json ->> 'deactivated') STORED, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (name) ); @@ -363,11 +317,9 @@ CREATE TABLE IF NOT EXISTS bot_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (name) ); @@ -377,11 +329,9 @@ CREATE TABLE IF NOT EXISTS role_entity ( name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, defaultRole BOOLEAN GENERATED ALWAYS AS ((json ->> 'defaultRole')::boolean) STORED, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (name) ); @@ -406,36 +356,31 @@ CREATE TABLE IF NOT EXISTS entity_usage ( -- Tag related tables -- CREATE TABLE IF NOT EXISTS tag_category ( - name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, json JSONB NOT NULL, -- JSON stores category information and does not store children - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - UNIQUE (name) -- Unique tag category name ); CREATE TABLE IF NOT EXISTS tag ( - fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, -- JSON stores all tag attributes and does not store children - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - UNIQUE (fullyQualifiedName) ); CREATE TABLE IF NOT EXISTS tag_usage ( - tagFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the tag - targetFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the entity instance or corresponding field - labelType SMALLINT, -- Type of tagging: manual, automated, propagated NOT NULL, derived - state SMALLINT NOT NULL, -- State of tagging: suggested or confirmed - source SMALLINT NOT NULL, -- Source of tag (either from TagCategory or Glossary) - + source SMALLINT NOT NULL, -- Source of the tag label + tagFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the tag + targetFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the entity instance or corresponding field + labelType SMALLINT NOT NULL, -- Type of tagging: manual, automated, propagated, derived + state SMALLINT NOT NULL, -- State of tagging: suggested or confirmed UNIQUE (source, tagFQN, targetFQN) ); @@ -456,7 +401,6 @@ CREATE TABLE IF NOT EXISTS webhook_entity ( name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, json JSON NOT NULL, - PRIMARY KEY (id), UNIQUE (name) -- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook @@ -466,11 +410,9 @@ CREATE TABLE IF NOT EXISTS glossary_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (name) ); @@ -479,11 +421,9 @@ CREATE TABLE IF NOT EXISTS glossary_term_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, - PRIMARY KEY (id), UNIQUE (fullyQualifiedName) ); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java index f8ff414faa8..82bae20ad0e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java @@ -52,9 +52,6 @@ public final class Entity { // Canonical entity name to corresponding EntityRepository map private static final Map> ENTITY_REPOSITORY_MAP = new HashMap<>(); - // Entity class to entity repository map - private static final Map, EntityRepository> CLASS_ENTITY_REPOSITORY_MAP = new HashMap<>(); - // Common field names public static final String FIELD_OWNER = "owner"; public static final String FIELD_NAME = "name"; @@ -140,7 +137,6 @@ public final class Entity { DAO_MAP.put(entity, dao); ENTITY_REPOSITORY_MAP.put(entity, entityRepository); CANONICAL_ENTITY_NAME_MAP.put(entity.toLowerCase(Locale.ROOT), entity); - CLASS_ENTITY_REPOSITORY_MAP.put(clazz, entityRepository); LOG.info( "Registering entity {} {} {} {}", clazz, diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/CatalogApplicationTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/CatalogApplicationTest.java index 6241b594bbe..74f70754136 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/CatalogApplicationTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/CatalogApplicationTest.java @@ -41,7 +41,6 @@ public abstract class CatalogApplicationTest { static { CollectionRegistry.addTestResource(webhookCallbackResource); - Fernet.getInstance().setFernetKey(FERNET_KEY_1); } @@ -54,6 +53,9 @@ public abstract class CatalogApplicationTest { SQL_CONTAINER = (JdbcDatabaseContainer) Class.forName(jdbcContainerClassName).getConstructor(String.class).newInstance(jdbcContainerImage); + SQL_CONTAINER.withReuse(true); + SQL_CONTAINER.withStartupTimeoutSeconds(240); + SQL_CONTAINER.withConnectTimeoutSeconds(240); SQL_CONTAINER.start(); final String migrationScripsLocation = @@ -84,11 +86,8 @@ public abstract class CatalogApplicationTest { @AfterAll public static void stopApplication() { - // If beforeAll causes and exception AfterAll still gets called before that exception is thrown. + // If BeforeAll causes and exception AfterAll still gets called before that exception is thrown. // If a NullPointerException is thrown during the cleanup of above it will eat the initial error - if (SQL_CONTAINER != null) { - SQL_CONTAINER.stop(); - } if (APP != null) { APP.after(); } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java index fd049a35387..f5878a25885 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java @@ -393,6 +393,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { }; // Test listing entities that include deleted, non-deleted, and all the entities + Random random = new Random(); for (String include : List.of("non-deleted", "all", "deleted")) { if (!supportsSoftDelete && include.equals("deleted")) { continue; @@ -405,9 +406,9 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { int totalRecords = allEntities.getData().size(); printEntities(allEntities); - // List entity with "limit" set from 1 to maxTables size + // List entity with "limit" set from 1 to maxTables size with random jumps (to reduce the test time) // Each time compare the returned list with allTables list to make sure right results are returned - for (int limit = 1; limit < maxEntities; limit++) { + for (int limit = 1; limit < maxEntities; limit += random.nextInt(5) + 1) { String after = null; String before; int pageCount = 0; @@ -416,7 +417,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { ResultList backwardPage; boolean foundDeleted = false; do { // For each limit (or page size) - forward scroll till the end - LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after); + LOG.debug("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after); forwardPage = listEntities(queryParams, limit, null, after, ADMIN_AUTH_HEADERS); foundDeleted = forwardPage.getData().stream().anyMatch(matchDeleted) || foundDeleted; after = forwardPage.getPaging().getAfter(); @@ -447,7 +448,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { indexInAllTables = totalRecords - limit - forwardPage.getData().size(); foundDeleted = false; do { - LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before); + LOG.debug("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before); forwardPage = listEntities(queryParams, limit, before, null, ADMIN_AUTH_HEADERS); foundDeleted = forwardPage.getData().stream().anyMatch(matchDeleted) || foundDeleted; printEntities(forwardPage); @@ -1819,8 +1820,8 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { } private void printEntities(ResultList list) { - list.getData().forEach(e -> LOG.info("{} {}", entityClass, getEntityInterface(e).getFullyQualifiedName())); - LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter()); + list.getData().forEach(e -> LOG.debug("{} {}", entityClass, getEntityInterface(e).getFullyQualifiedName())); + LOG.debug("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter()); } public void assertEntityDeleted(EntityInterface entityInterface, boolean hardDelete) { diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java index 549bcddce0c..409aa1aea46 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java @@ -15,11 +15,9 @@ package org.openmetadata.catalog.resources.policies; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.fail; import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS; import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE; -import static org.openmetadata.catalog.util.TestUtils.assertEntityPagination; import static org.openmetadata.catalog.util.TestUtils.assertListNotNull; import static org.openmetadata.catalog.util.TestUtils.assertListNull; import static org.openmetadata.catalog.util.TestUtils.assertResponse; @@ -31,9 +29,7 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.stream.Collectors; -import javax.ws.rs.client.WebTarget; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.BeforeAll; @@ -178,90 +174,6 @@ public class PolicyResourceTest extends EntityResourceTest getEntityName(test))); } - @Test - void get_PolicyListWithInvalidLimitOffset_4xx() { - // Limit must be >= 1 and <= 1000,000 - assertResponse( - () -> listPolicies(null, -1, null, null, ADMIN_AUTH_HEADERS), - BAD_REQUEST, - "[query param limit must be greater than or equal to 0]"); - - assertResponse( - () -> listPolicies(null, 1000001, null, null, ADMIN_AUTH_HEADERS), - BAD_REQUEST, - "[query param limit must be less than or equal to 1000000]"); - } - - @Test - void get_PolicyListWithInvalidPaginationCursors_4xx() { - // Passing both before and after cursors is invalid - assertResponse( - () -> listPolicies(null, 1, "", "", ADMIN_AUTH_HEADERS), - BAD_REQUEST, - "Only one of before or after query parameter allowed"); - } - - @Test - void get_PolicyListWithValidLimitOffset_4xx(TestInfo test) throws HttpResponseException { - // Create a large number of Policies - int maxPolicies = 40; - for (int i = 0; i < maxPolicies; i++) { - createEntity(createRequest(test, i), ADMIN_AUTH_HEADERS); - } - - // List all Policies - PolicyList allPolicies = listPolicies(null, 1000000, null, null, ADMIN_AUTH_HEADERS); - int totalRecords = allPolicies.getData().size(); - printPolicies(allPolicies); - - // List limit number Policies at a time at various offsets and ensure right results are returned - for (int limit = 1; limit < maxPolicies; limit++) { - String after = null; - String before; - int pageCount = 0; - int indexInAllPolicies = 0; - PolicyList forwardPage; - PolicyList backwardPage; - do { // For each limit (or page size) - forward scroll till the end - LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after); - forwardPage = listPolicies(null, limit, null, after, ADMIN_AUTH_HEADERS); - printPolicies(forwardPage); - after = forwardPage.getPaging().getAfter(); - before = forwardPage.getPaging().getBefore(); - assertEntityPagination(allPolicies.getData(), forwardPage, limit, indexInAllPolicies); - - if (pageCount == 0) { // CASE 0 - First page is being returned. Therefore, before cursor is null - assertNull(before); - } else { - // Make sure scrolling back based on before cursor returns the correct result - backwardPage = listPolicies(null, limit, before, null, ADMIN_AUTH_HEADERS); - assertEntityPagination(allPolicies.getData(), backwardPage, limit, (indexInAllPolicies - limit)); - } - - indexInAllPolicies += forwardPage.getData().size(); - pageCount++; - } while (after != null); - - // We have now reached the last page - test backward scroll till the beginning - pageCount = 0; - indexInAllPolicies = totalRecords - limit - forwardPage.getData().size(); - do { - LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before); - forwardPage = listPolicies(null, limit, before, null, ADMIN_AUTH_HEADERS); - printPolicies(forwardPage); - before = forwardPage.getPaging().getBefore(); - assertEntityPagination(allPolicies.getData(), forwardPage, limit, indexInAllPolicies); - pageCount++; - indexInAllPolicies -= forwardPage.getData().size(); - } while (before != null); - } - } - - private void printPolicies(PolicyList list) { - list.getData().forEach(Policy -> LOG.info("DB {}", Policy.getFullyQualifiedName())); - LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter()); - } - @Test void patch_PolicyAttributes_200_ok(TestInfo test) throws IOException { Policy policy = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS).withLocation(null); @@ -305,46 +217,22 @@ public class PolicyResourceTest extends EntityResourceTest String fields = ""; policy = byName - ? getPolicyByName(policy.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS) - : getPolicy(policy.getId(), fields, ADMIN_AUTH_HEADERS); + ? getEntityByName(policy.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS) + : getEntity(policy.getId(), fields, ADMIN_AUTH_HEADERS); assertListNull(policy.getOwner(), policy.getLocation()); // .../policies?fields=owner,displayName,policyUrl fields = "owner,location"; policy = byName - ? getPolicyByName(policy.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS) - : getPolicy(policy.getId(), fields, ADMIN_AUTH_HEADERS); + ? getEntityByName(policy.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS) + : getEntity(policy.getId(), fields, ADMIN_AUTH_HEADERS); // Field location is set during creation - tested elsewhere assertListNotNull(policy.getOwner() /*, policy.getLocation()*/); // Checks for other owner, tags, and followers is done in the base class return getEntityInterface(policy); } - public static Policy getPolicy(UUID id, String fields, Map authHeaders) throws HttpResponseException { - WebTarget target = getResource("policies/" + id); - target = fields != null ? target.queryParam("fields", fields) : target; - return TestUtils.get(target, Policy.class, authHeaders); - } - - public static Policy getPolicyByName(String fqn, String fields, Map authHeaders) - throws HttpResponseException { - WebTarget target = getResource("policies/name/" + fqn); - target = fields != null ? target.queryParam("fields", fields) : target; - return TestUtils.get(target, Policy.class, authHeaders); - } - - public static PolicyList listPolicies( - String fields, Integer limitParam, String before, String after, Map authHeaders) - throws HttpResponseException { - WebTarget target = getResource("policies"); - target = fields != null ? target.queryParam("fields", fields) : target; - target = limitParam != null ? target.queryParam("limit", limitParam) : target; - target = before != null ? target.queryParam("before", before) : target; - target = after != null ? target.queryParam("after", after) : target; - return TestUtils.get(target, PolicyList.class, authHeaders); - } - private CreatePolicy createAccessControlPolicyWithRules(String name, List rules) { return new CreatePolicy() .withName(name)