From 07b039b8b8c88070145e4adeeb80923409160ca2 Mon Sep 17 00:00:00 2001 From: Siddhant <86899184+Siddhanttimeline@users.noreply.github.com> Date: Fri, 11 Oct 2024 19:21:12 +0530 Subject: [PATCH] Migrations fixes for #17967 (#18232) * fix migrations. * fix logging --- .../mysql/postDataMigrationSQLScript.sql | 17 +--- .../postgres/postDataMigrationSQLScript.sql | 17 +--- .../migration/mysql/v160/Migration.java | 2 + .../migration/postgres/v160/Migration.java | 2 + .../migration/utils/v160/MigrationUtil.java | 90 +++++++++++++++++++ 5 files changed, 96 insertions(+), 32 deletions(-) diff --git a/bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql index 0ad7d922a4e..b22e4c83497 100644 --- a/bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql +++ b/bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql @@ -42,19 +42,4 @@ SET json = JSON_REMOVE(json, '$.testCaseResult'); UPDATE installed_apps SET json = JSON_SET(json, '$.supportsInterrupt', true) where name = 'SearchIndexingApplication'; UPDATE apps_marketplace SET json = JSON_SET(json, '$.supportsInterrupt', true) where name = 'SearchIndexingApplication'; -ALTER TABLE apps_extension_time_series ADD COLUMN appName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.appName') STORED NOT NULL; - --- Update serviceType in api_endpoint_entity table -UPDATE api_endpoint_entity -SET json = JSON_SET(json, '$.serviceType', 'Rest') -WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'REST'; - --- Update serviceType in api_service_entity table -UPDATE api_service_entity -SET json = JSON_SET(json, '$.serviceType', 'Rest') -WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'REST'; - --- Update serviceType in api_collection_entity table -UPDATE api_collection_entity -SET json = JSON_SET(json, '$.serviceType', 'Rest') -WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'REST'; \ No newline at end of file +ALTER TABLE apps_extension_time_series ADD COLUMN appName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.appName') STORED NOT NULL; \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql index 0ea510e4231..deb05a69b53 100644 --- a/bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql +++ b/bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql @@ -59,19 +59,4 @@ SET json = jsonb_set( ) where name = 'SearchIndexingApplication'; -ALTER TABLE apps_extension_time_series ADD COLUMN appName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'appName') STORED NOT NULL; - --- Update serviceType in api_endpoint_entity table -UPDATE api_endpoint_entity -SET json = jsonb_set(json, '{serviceType}', '"Rest"') -WHERE jsonb_extract_path_text(json, 'serviceType') = 'REST'; - --- Update serviceType in api_service_entity table -UPDATE api_service_entity -SET json = jsonb_set(json, '{serviceType}', '"Rest"') -WHERE jsonb_extract_path_text(json, 'serviceType') = 'REST'; - --- Update serviceType in api_collection_entity table -UPDATE api_collection_entity -SET json = jsonb_set(json, '{serviceType}', '"Rest"') -WHERE jsonb_extract_path_text(json, 'serviceType') = 'REST'; \ No newline at end of file +ALTER TABLE apps_extension_time_series ADD COLUMN appName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'appName') STORED NOT NULL; \ No newline at end of file diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java index 9be8ca5e22f..bf3d4769338 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java @@ -1,6 +1,7 @@ package org.openmetadata.service.migration.mysql.v160; import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addAppExtensionName; +import static org.openmetadata.service.migration.utils.v160.MigrationUtil.migrateServiceTypesAndConnections; import lombok.SneakyThrows; import org.openmetadata.service.migration.api.MigrationProcessImpl; @@ -16,5 +17,6 @@ public class Migration extends MigrationProcessImpl { @SneakyThrows public void runDataMigration() { addAppExtensionName(handle, collectionDAO, authenticationConfiguration, false); + migrateServiceTypesAndConnections(handle, false); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java index d3f0651fece..ca1f9547f54 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java @@ -1,6 +1,7 @@ package org.openmetadata.service.migration.postgres.v160; import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addAppExtensionName; +import static org.openmetadata.service.migration.utils.v160.MigrationUtil.migrateServiceTypesAndConnections; import lombok.SneakyThrows; import org.openmetadata.service.migration.api.MigrationProcessImpl; @@ -16,5 +17,6 @@ public class Migration extends MigrationProcessImpl { @SneakyThrows public void runDataMigration() { addAppExtensionName(handle, collectionDAO, authenticationConfiguration, true); + migrateServiceTypesAndConnections(handle, true); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java index 2e823400089..8c1608f5195 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java @@ -80,4 +80,94 @@ public class MigrationUtil { } update.bind("appId", app.getId().toString()).bind("appName", app.getName()).execute(); } + + public static void migrateServiceTypesAndConnections(Handle handle, boolean postgresql) { + LOG.info("Starting service type and connection type migrations"); + try { + migrateServiceTypeInApiEndPointServiceType(handle, postgresql); + migrateServiceTypeInApiServiceEntity(handle, postgresql); + migrateConnectionTypeInApiServiceEntity(handle, postgresql); + migrateServiceTypeInApiCollectionEntity(handle, postgresql); + LOG.info("Successfully completed service type and connection type migrations"); + } catch (Exception e) { + LOG.error("Error occurred during migration", e); + } + } + + private static void migrateServiceTypeInApiEndPointServiceType( + Handle handle, boolean postgresql) { + LOG.info("Starting migrateServiceTypeInApiEndPointServiceType"); + String query; + + if (postgresql) { + query = + "UPDATE api_endpoint_entity SET json = jsonb_set(json, '{serviceType}', '\"Rest\"', false) WHERE jsonb_extract_path_text(json, 'serviceType') = 'REST'"; + } else { + query = + "UPDATE api_endpoint_entity SET json = JSON_SET(json, '$.serviceType', 'Rest') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'REST'"; + } + + try { + handle.execute(query); + } catch (Exception e) { + LOG.error("Error updating", e); + } + } + + private static void migrateServiceTypeInApiServiceEntity(Handle handle, boolean postgresql) { + LOG.info("Starting migrateServiceTypeInApiServiceEntity"); + + String query; + if (postgresql) { + query = + "UPDATE api_service_entity SET json = jsonb_set(json, '{serviceType}', '\"Rest\"', false) WHERE jsonb_extract_path_text(json, 'serviceType') = 'REST'"; + } else { + query = + "UPDATE api_service_entity SET json = JSON_SET(json, '$.serviceType', 'Rest') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'REST'"; + } + + try { + handle.execute(query); + } catch (Exception e) { + LOG.error("Error updating", e); + } + } + + private static void migrateServiceTypeInApiCollectionEntity(Handle handle, boolean postgresql) { + LOG.info("Starting runApiCollectionEntityServiceTypeDataMigrations"); + + String query; + if (postgresql) { + query = + "UPDATE api_collection_entity SET json = jsonb_set(json, '{serviceType}', '\"Rest\"', false) WHERE jsonb_extract_path_text(json, 'serviceType') = 'REST'"; + } else { + query = + "UPDATE api_collection_entity SET json = JSON_SET(json, '$.serviceType', 'Rest') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'REST'"; + } + + try { + handle.execute(query); + } catch (Exception e) { + LOG.error("Error updating", e); + } + } + + private static void migrateConnectionTypeInApiServiceEntity(Handle handle, boolean postgresql) { + LOG.info("Starting runApiServiceEntityConnectionTypeMigrate"); + + String query; + if (postgresql) { + query = + "UPDATE api_service_entity SET json = jsonb_set(json, '{connection,config,type}', '\"Rest\"', false) WHERE jsonb_extract_path_text(json, 'connection', 'config', 'type') = 'REST'"; + } else { + query = + "UPDATE api_service_entity SET json = JSON_SET(json, '$.connection.config.type', 'Rest') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.connection.config.type')) = 'REST'"; + } + + try { + handle.execute(query); + } catch (Exception e) { + LOG.error("Error updating", e); + } + } }