yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-13 09:23:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR - Migration - Automator multiple ownership (#17248)

Browse Source
This commit is contained in:
Pere Miquel Brull 2024-08-01 10:16:26 +02:00 committed by GitHub
parent b6905b4bdd
commit 57e6ad57cd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 81 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v150/Migration.java
View File

@ -2,6 +2,7 @@ package org.openmetadata.service.migration.mysql.v150;
import static org.openmetadata.service.migration.utils.v150.MigrationUtil.createSystemDICharts; import static org.openmetadata.service.migration.utils.v150.MigrationUtil.createSystemDICharts;
import static org.openmetadata.service.migration.utils.v150.MigrationUtil.deleteLegacyDataInsightPipelines; import static org.openmetadata.service.migration.utils.v150.MigrationUtil.deleteLegacyDataInsightPipelines;
import static org.openmetadata.service.migration.utils.v150.MigrationUtil.migrateAutomatorOwner;
import static org.openmetadata.service.migration.utils.v150.MigrationUtil.migrateTestCaseDimension; import static org.openmetadata.service.migration.utils.v150.MigrationUtil.migrateTestCaseDimension;
import lombok.SneakyThrows; import lombok.SneakyThrows;
@ -20,5 +21,6 @@ public class Migration extends MigrationProcessImpl {
migrateTestCaseDimension(handle, collectionDAO); migrateTestCaseDimension(handle, collectionDAO);
createSystemDICharts(); createSystemDICharts();
deleteLegacyDataInsightPipelines(pipelineServiceClient); deleteLegacyDataInsightPipelines(pipelineServiceClient);
migrateAutomatorOwner(handle, collectionDAO);
} }
} }

2
openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v150/Migration.java
View File

@ -2,6 +2,7 @@ package org.openmetadata.service.migration.postgres.v150;
import static org.openmetadata.service.migration.utils.v150.MigrationUtil.createSystemDICharts; import static org.openmetadata.service.migration.utils.v150.MigrationUtil.createSystemDICharts;
import static org.openmetadata.service.migration.utils.v150.MigrationUtil.deleteLegacyDataInsightPipelines; import static org.openmetadata.service.migration.utils.v150.MigrationUtil.deleteLegacyDataInsightPipelines;
import static org.openmetadata.service.migration.utils.v150.MigrationUtil.migrateAutomatorOwner;
import static org.openmetadata.service.migration.utils.v150.MigrationUtil.migrateTestCaseDimension; import static org.openmetadata.service.migration.utils.v150.MigrationUtil.migrateTestCaseDimension;
import lombok.SneakyThrows; import lombok.SneakyThrows;
@ -20,5 +21,6 @@ public class Migration extends MigrationProcessImpl {
migrateTestCaseDimension(handle, collectionDAO); migrateTestCaseDimension(handle, collectionDAO);
createSystemDICharts(); createSystemDICharts();
deleteLegacyDataInsightPipelines(pipelineServiceClient); deleteLegacyDataInsightPipelines(pipelineServiceClient);
migrateAutomatorOwner(handle, collectionDAO);
} }
} }

77
openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v150/MigrationUtil.java
View File

@ -3,6 +3,11 @@ package org.openmetadata.service.migration.utils.v150;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import javax.json.Json;
import javax.json.JsonArray;
import javax.json.JsonArrayBuilder;
import javax.json.JsonObject;
import javax.json.JsonObjectBuilder;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Handle; import org.jdbi.v3.core.Handle;
import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart;
@ -23,6 +28,78 @@ import org.openmetadata.service.util.JsonUtils;
@Slf4j @Slf4j
public class MigrationUtil { public class MigrationUtil {
private static final String QUERY_AUTOMATOR =
"SELECT json FROM ingestion_pipeline_entity where appType = 'Automator'";
private static final String ADD_OWNER_ACTION = "AddOwnerAction";
/**
* We need to update the `AddOwnerAction` action in the automator to have a list of owners
*/
public static void migrateAutomatorOwner(Handle handle, CollectionDAO collectionDAO) {
try {
handle
.createQuery(QUERY_AUTOMATOR)
.mapToMap()
.forEach(
row -> {
try {
// Prepare the current json objects
JsonObject json = JsonUtils.readJson((String) row.get("json")).asJsonObject();
JsonObject sourceConfig = json.getJsonObject("sourceConfig");
JsonObject config = sourceConfig.getJsonObject("config");
JsonObject appConfig = config.getJsonObject("appConfig");
JsonArray actions = appConfig.getJsonArray("actions");
JsonArrayBuilder updatedActions = Json.createArrayBuilder();
// update the AddOwnerAction payloads to have a list of owners
actions.forEach(
action -> {
JsonObject actionObj = (JsonObject) action;
if (ADD_OWNER_ACTION.equals(actionObj.getString("type"))) {
JsonObject owner = actionObj.getJsonObject("owner");
JsonArrayBuilder owners = Json.createArrayBuilder();
owners.add(owner);
actionObj =
Json.createObjectBuilder(actionObj)
.add("owners", owners)
.remove("owner")
.build();
}
updatedActions.add(actionObj);
});
// Recreate the json object
JsonObjectBuilder updatedAppConfig =
Json.createObjectBuilder(appConfig).add("actions", updatedActions);
JsonObjectBuilder updatedConfig =
Json.createObjectBuilder(config).add("appConfig", updatedAppConfig);
JsonObjectBuilder updatedSourceConfig =
Json.createObjectBuilder(sourceConfig).add("config", updatedConfig);
JsonObject finalJsonObject =
Json.createObjectBuilder(json)
.add("sourceConfig", updatedSourceConfig)
.build();
// Update the Ingestion Pipeline
IngestionPipeline ingestionPipeline =
JsonUtils.readValue(finalJsonObject.toString(), IngestionPipeline.class);
collectionDAO.ingestionPipelineDAO().update(ingestionPipeline);
} catch (Exception ex) {
LOG.warn(String.format("Error updating automator [%s] due to [%s]", row, ex));
}
});
} catch (Exception ex) {
LOG.warn("Error running the automator migration ", ex);
}
}
public static void deleteLegacyDataInsightPipelines( public static void deleteLegacyDataInsightPipelines(
PipelineServiceClientInterface pipelineServiceClient) { PipelineServiceClientInterface pipelineServiceClient) {
// Delete Data Insights Pipeline // Delete Data Insights Pipeline