From 3a7f7486d0c2ec08845df7e877f1d1efeb18a5f1 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Fri, 29 Sep 2023 05:19:49 -0700 Subject: [PATCH] Add indexes to name column for efficient ORDER BY name queries (#13383) --- .../native/1.2.0/mysql/schemaChanges.sql | 57 ++++++++++++++++-- .../native/1.2.0/postgres/schemaChanges.sql | 59 +++++++++++++++---- 2 files changed, 100 insertions(+), 16 deletions(-) diff --git a/bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql index dd6f9532aca..9fdf786af1f 100644 --- a/bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql @@ -11,7 +11,8 @@ CREATE TABLE IF NOT EXISTS domain_entity ( updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, PRIMARY KEY (id), - UNIQUE (fqnHash) + UNIQUE (fqnHash), + INDEX (name) ); -- create data product entity table @@ -23,7 +24,8 @@ CREATE TABLE IF NOT EXISTS data_product_entity ( updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, PRIMARY KEY (id), - UNIQUE (fqnHash) + UNIQUE (fqnHash), + INDEX (name) ); -- create search service entity @@ -37,7 +39,8 @@ CREATE TABLE IF NOT EXISTS search_service_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE (nameHash) + UNIQUE (nameHash), + INDEX (name) ); -- create search index entity @@ -50,7 +53,8 @@ CREATE TABLE IF NOT EXISTS search_index_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE (fqnHash) + UNIQUE (fqnHash), + INDEX (name) ); -- We were hardcoding retries to 0. Since we are now using the IngestionPipeline to set them, keep existing ones to 0. @@ -69,7 +73,8 @@ CREATE TABLE IF NOT EXISTS stored_procedure_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), PRIMARY KEY (id), - UNIQUE (fqnHash) + UNIQUE (fqnHash), + INDEX (name) ); ALTER TABLE entity_relationship ADD INDEX from_entity_type_index(fromId, fromEntity), ADD INDEX to_entity_type_index(toId, toEntity); @@ -111,4 +116,44 @@ SET json = JSON_INSERT( WHERE serviceType = 'DomoPipeline'; -- Query Entity supports service, which requires FQN for name -ALTER TABLE query_entity CHANGE COLUMN nameHash fqnHash VARCHAR(256); \ No newline at end of file +ALTER TABLE query_entity CHANGE COLUMN nameHash fqnHash VARCHAR(256); +ALTER TABLE bot_entity add index bot_entity_name_index(name); +ALTER TABLE chart_entity add index chart_entity_name_index(name); +ALTER TABLE classification add index classification_entity_name_index(name); +ALTER TABLE storage_container_entity add index storage_container_entity_name_index(name); +ALTER TABLE dashboard_data_model_entity add index dashboard_data_model_entity_name_index(name); +ALTER TABLE dashboard_entity add index dashboard_entity_name_index(name); +ALTER TABLE dashboard_service_entity add index dashboard_service_entity_name_index(name); +ALTER TABLE data_insight_chart add index data_insight_name_index(name); +ALTER TABLE database_entity add index database_entity_name_index(name); +ALTER TABLE database_schema_entity add index database_schema_entity_name_index(name); +ALTER TABLE dbservice_entity add index dbservice_entity_name_index(name); +ALTER TABLE event_subscription_entity add index event_subscription_entity_name_index(name); +ALTER TABLE glossary_entity add index glossary_entity_name_index(name); +ALTER TABLE glossary_term_entity add index glossary_term_entity_name_index(name); +ALTER TABLE ingestion_pipeline_entity add index ingestion_pipeline_entity_name_index(name); +ALTER TABLE kpi_entity add index kpi_entity_name_index(name); +ALTER TABLE messaging_service_entity add index messaing_service_entity_name_index(name); +ALTER TABLE metadata_service_entity add index metadata_service_entity_name_index(name); +ALTER TABLE metric_entity add index metric_entity_name_index(name); +ALTER TABLE ml_model_entity add index ml_model_entity_name_index(name); +ALTER TABLE mlmodel_service_entity add index mlmodel_service_entity_name_index(name); +ALTER TABLE pipeline_entity add index pipeline_entity_name_index(name); +ALTER TABLE pipeline_service_entity add index pipeline_service_entity_name_index(name); +ALTER TABLE policy_entity add index policy_entity_name_index(name); +ALTER TABLE query_entity add index query_entity_name_index(name); +ALTER TABLE report_entity add index report_entity_name_index(name); +ALTER TABLE role_entity add index role_entity_name_index(name); +ALTER TABLE storage_service_entity add index storage_service_entity_name_index(name); +ALTER TABLE table_entity add index table_entity_name_index(name); +ALTER TABLE tag add index tag_entity_name_index(name); +ALTER TABLE team_entity add index team_entity_name_index(name); +ALTER TABLE test_case add index test_case_name_index(name); +ALTER TABLE test_connection_definition add index test_connection_definition_name_index(name); +ALTER TABLE test_definition add index test_definition_name_index(name); +ALTER TABLE test_suite add index test_suite_name_index(name); +ALTER TABLE topic_entity add index topic_entity_name_index(name); +ALTER TABLE type_entity add index type_entity_name_index(name); +ALTER TABLE user_entity add index user_entity_name_index(name); +ALTER TABLE web_analytic_event add index web_analytic_event_name_index(name); +ALTER TABLE automations_workflow add index automations_workflow_name_index(name); diff --git a/bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql index 8896b4a6066..686fd78f652 100644 --- a/bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql @@ -70,15 +70,6 @@ CREATE TABLE IF NOT EXISTS stored_procedure_entity ( UNIQUE (fqnHash) ); --- create entity extension table for table entity -CREATE TABLE IF NOT EXISTS table_entity_extension ( - id VARCHAR(36) NOT NULL, -- ID of the from entity - extension VARCHAR(256) NOT NULL, -- Extension name same as entity.fieldName - jsonSchema VARCHAR(256) NOT NULL, -- Schema used for generating JSON - json JSONB NOT NULL, - PRIMARY KEY (id, extension) -); - -- Add index on fromId and fromEntity columns CREATE INDEX from_entity_type_index ON entity_relationship (fromId, fromEntity); @@ -131,4 +122,52 @@ SET json = jsonb_set( WHERE serviceType = 'DomoPipeline'; -- Query Entity supports service, which requires FQN for name -ALTER TABLE query_entity RENAME COLUMN nameHash TO fqnHash; \ No newline at end of file +ALTER TABLE query_entity RENAME COLUMN nameHash TO fqnHash; + +CREATE INDEX idx_name_query_entity ON query_entity (name); +CREATE INDEX idx_name_bot_entity ON bot_entity (name); +CREATE INDEX idx_name_chart_entity ON chart_entity (name); +CREATE INDEX idx_name_classification_entity ON classification (name); +CREATE INDEX idx_name_storage_container_entity ON storage_container_entity (name); +CREATE INDEX idx_name_dashboard_data_model_entity ON dashboard_data_model_entity (name); +CREATE INDEX idx_name_dashboard_entity ON dashboard_entity (name); +CREATE INDEX idx_name_dashboard_service_entity ON dashboard_service_entity (name); +CREATE INDEX idx_name_dashboard_insight_chart ON data_insight_chart (name); +CREATE INDEX idx_name_database_entity ON database_entity (name); +CREATE INDEX idx_name_database_schema_entity ON database_schema_entity (name); +CREATE INDEX idx_name_db_service_entity ON dbservice_entity (name); +CREATE INDEX idx_name_event_subscription_entity ON event_subscription_entity (name); +CREATE INDEX idx_name_glossary_entity ON glossary_entity (name); +CREATE INDEX idx_name_glossary_term_entity ON glossary_term_entity (name); +CREATE INDEX idx_name_ingestion_pipeline_entity ON ingestion_pipeline_entity (name); +CREATE INDEX idx_name_kpi_entity ON kpi_entity (name); +CREATE INDEX idx_messaging_service_name_entity ON messaging_service_entity (name); +CREATE INDEX idx_metadata_service_name_entity ON metadata_service_entity (name); +CREATE INDEX idx_metric_name_entity ON metric_entity (name); +CREATE INDEX idx_ml_model_name_entity ON ml_model_entity (name); +CREATE INDEX idx_ml_model_service_name_entity ON mlmodel_service_entity (name); +CREATE INDEX idx_pipeline_name_entity ON pipeline_entity (name); +CREATE INDEX idx_pipeline_service_name_entity ON pipeline_service_entity (name); +CREATE INDEX idx_name_policy_entity ON policy_entity (name); +CREATE INDEX idx_name_query_entity ON query_entity (name); +CREATE INDEX idx_name_report_entity ON reprot_entity (name); +CREATE INDEX idx_name_role_entity ON role_entity (name); +CREATE INDEX idx_name_storage_service_entity ON storage_service_entity (name); +CREATE INDEX idx_name_table_entity ON table_entity (name); +CREATE INDEX idx_name_tag_entity ON tag (name); +CREATE INDEX idx_name_team_entity ON team_entity (name); +CREATE INDEX idx_name_test_case ON test_case (name); +CREATE INDEX idx_name_test_connection_definition ON test_connection_definition (name); +CREATE INDEX idx_name_test_definition ON test_definition (name); +CREATE INDEX idx_name_test_suite ON test_suite (name); +CREATE INDEX idx_name_topic_entity ON topic_entity (name); +CREATE INDEX idx_name_type_entity ON type_entity (name); +CREATE INDEX idx_name_user_entity ON user_entity (name); +CREATE INDEX idx_name_web_analytic_event ON web_analytic_event (name); +CREATE INDEX idx_name_automations_workflow ON automations_workflow (name); +CREATE INDEX idx_name_domain_entity ON domain_entity (name); +CREATE INDEX idx_name_data_product_entity ON data_product_entity (name); +CREATE INDEX idx_name_data_search_service_entity ON search_service_entity (name); +CREATE INDEX idx_name_data_search_index_entity ON search_index_entity (name); +CREATE INDEX idx_name_data_stored_procedure_entity ON stored_procedure_entity (name); +