From 5a83cfe640ca1297ec717409cc8fee78154d6d35 Mon Sep 17 00:00:00 2001 From: IceS2 Date: Tue, 20 May 2025 08:02:02 +0200 Subject: [PATCH] MINOR: Add missing workflow config status migrations (#21286) * Add missing workflow config status migrations * Moved to Java based migrations to redeploy pipeline * Moved migrations to Java. Updated Listener based on config * Improved update on the migrations (cherry picked from commit b761efbfd33e1888dc59e0ef7fe636a9a98aa8c5) --- .../native/1.7.1/mysql/schemaChanges.sql | 2 +- .../native/1.7.1/postgres/schemaChanges.sql | 2 +- .../workflows/elements/NodeInterface.java | 9 ++++-- .../CheckEntityAttributesTask.java | 3 +- .../SetEntityCertificationTask.java | 3 +- .../SetGlossaryTermStatusTask.java | 3 +- .../CreateAndRunIngestionPipelineTask.java | 3 +- .../automatedTask/runApp/RunAppTask.java | 3 +- .../nodes/userTask/UserApprovalTask.java | 3 +- .../migration/mysql/v171/Migration.java | 5 ++++ .../migration/postgres/v171/Migration.java | 5 ++++ .../migration/utils/v171/MigrationUtil.java | 29 +++++++++++++++++++ 12 files changed, 59 insertions(+), 11 deletions(-) diff --git a/bootstrap/sql/migrations/native/1.7.1/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.7.1/mysql/schemaChanges.sql index eecc9128ab4..02f72c1bda0 100644 --- a/bootstrap/sql/migrations/native/1.7.1/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.7.1/mysql/schemaChanges.sql @@ -32,4 +32,4 @@ SET json = JSON_SET( '$.runtime.enabled', true ) -WHERE name = 'AutoPilotApplication'; \ No newline at end of file +WHERE name = 'AutoPilotApplication'; diff --git a/bootstrap/sql/migrations/native/1.7.1/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.7.1/postgres/schemaChanges.sql index 57d3a431d64..f2bd776c59d 100644 --- a/bootstrap/sql/migrations/native/1.7.1/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.7.1/postgres/schemaChanges.sql @@ -38,4 +38,4 @@ SET json = jsonb_set( '{runtime,enabled}', 'true' ) -where name = 'AutoPilotApplication'; \ No newline at end of file +where name = 'AutoPilotApplication'; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeInterface.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeInterface.java index 85a5dc74e5b..ec1fd36990f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeInterface.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeInterface.java @@ -70,7 +70,8 @@ public interface NodeInterface { endEvent.getExecutionListeners().add(listener); } - default BoundaryEvent getRuntimeExceptionBoundaryEvent(Activity activity) { + default BoundaryEvent getRuntimeExceptionBoundaryEvent( + Activity activity, Boolean storeStageStatus) { ErrorEventDefinition runtimeExceptionDefinition = new ErrorEventDefinition(); runtimeExceptionDefinition.setErrorCode(WORKFLOW_RUNTIME_EXCEPTION); @@ -80,8 +81,10 @@ public interface NodeInterface { runtimeExceptionBoundaryEvent.addEventDefinition(runtimeExceptionDefinition); runtimeExceptionBoundaryEvent.setAttachedToRef(activity); - for (FlowableListener listener : getWorkflowInstanceStageListeners(List.of("end"))) { - runtimeExceptionBoundaryEvent.getExecutionListeners().add(listener); + if (storeStageStatus) { + for (FlowableListener listener : getWorkflowInstanceStageListeners(List.of("end"))) { + runtimeExceptionBoundaryEvent.getExecutionListeners().add(listener); + } } return runtimeExceptionBoundaryEvent; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CheckEntityAttributesTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CheckEntityAttributesTask.java index f3316cec3d5..17ed9ea64fc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CheckEntityAttributesTask.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CheckEntityAttributesTask.java @@ -55,7 +55,8 @@ public class CheckEntityAttributesTask implements NodeInterface { attachWorkflowInstanceStageListeners(subProcess); } - this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess); + this.runtimeExceptionBoundaryEvent = + getRuntimeExceptionBoundaryEvent(subProcess, config.getStoreStageStatus()); this.subProcess = subProcess; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetEntityCertificationTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetEntityCertificationTask.java index 01025f9637b..d75b8687b19 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetEntityCertificationTask.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetEntityCertificationTask.java @@ -58,7 +58,8 @@ public class SetEntityCertificationTask implements NodeInterface { attachWorkflowInstanceStageListeners(subProcess); } - this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess); + this.runtimeExceptionBoundaryEvent = + getRuntimeExceptionBoundaryEvent(subProcess, config.getStoreStageStatus()); this.subProcess = subProcess; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetGlossaryTermStatusTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetGlossaryTermStatusTask.java index 54f0a72423a..33515f16e52 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetGlossaryTermStatusTask.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetGlossaryTermStatusTask.java @@ -55,7 +55,8 @@ public class SetGlossaryTermStatusTask implements NodeInterface { attachWorkflowInstanceStageListeners(subProcess); } - this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess); + this.runtimeExceptionBoundaryEvent = + getRuntimeExceptionBoundaryEvent(subProcess, config.getStoreStageStatus()); this.subProcess = subProcess; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipeline/CreateAndRunIngestionPipelineTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipeline/CreateAndRunIngestionPipelineTask.java index 47322354cd9..848cdaad49a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipeline/CreateAndRunIngestionPipelineTask.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipeline/CreateAndRunIngestionPipelineTask.java @@ -86,7 +86,8 @@ public class CreateAndRunIngestionPipelineTask implements NodeInterface { attachWorkflowInstanceStageListeners(subProcess); } - this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess); + this.runtimeExceptionBoundaryEvent = + getRuntimeExceptionBoundaryEvent(subProcess, config.getStoreStageStatus()); this.subProcess = subProcess; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppTask.java index c5779562e56..0a740ebba06 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppTask.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppTask.java @@ -56,7 +56,8 @@ public class RunAppTask implements NodeInterface { attachWorkflowInstanceStageListeners(subProcess); } - this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess); + this.runtimeExceptionBoundaryEvent = + getRuntimeExceptionBoundaryEvent(subProcess, config.getStoreStageStatus()); this.subProcess = subProcess; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/UserApprovalTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/UserApprovalTask.java index ed615fb28b5..b4e083d2c48 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/UserApprovalTask.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/UserApprovalTask.java @@ -103,7 +103,8 @@ public class UserApprovalTask implements NodeInterface { attachWorkflowInstanceStageListeners(subProcess); } - this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess); + this.runtimeExceptionBoundaryEvent = + getRuntimeExceptionBoundaryEvent(subProcess, config.getStoreStageStatus()); this.subProcess = subProcess; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v171/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v171/Migration.java index 084a5bd2217..72b104a2643 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v171/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v171/Migration.java @@ -1,6 +1,7 @@ package org.openmetadata.service.migration.mysql.v171; import static org.openmetadata.service.migration.utils.v171.MigrationUtil.updateServiceCharts; +import static org.openmetadata.service.migration.utils.v171.MigrationUtil.updateWorkflowDefinitions; import lombok.SneakyThrows; import org.openmetadata.service.migration.api.MigrationProcessImpl; @@ -16,5 +17,9 @@ public class Migration extends MigrationProcessImpl { @SneakyThrows public void runDataMigration() { updateServiceCharts(); + + // Updating WorkflowDefinition + initializeWorkflowHandler(); + updateWorkflowDefinitions(); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v171/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v171/Migration.java index 3ce7d2c79de..852f8c30ff1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v171/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v171/Migration.java @@ -1,6 +1,7 @@ package org.openmetadata.service.migration.postgres.v171; import static org.openmetadata.service.migration.utils.v171.MigrationUtil.updateServiceCharts; +import static org.openmetadata.service.migration.utils.v171.MigrationUtil.updateWorkflowDefinitions; import lombok.SneakyThrows; import org.openmetadata.service.migration.api.MigrationProcessImpl; @@ -16,5 +17,9 @@ public class Migration extends MigrationProcessImpl { @SneakyThrows public void runDataMigration() { updateServiceCharts(); + + // Updating WorkflowDefinition + initializeWorkflowHandler(); + updateWorkflowDefinitions(); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v171/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v171/MigrationUtil.java index 690f98e2658..bddc4de75d4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v171/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v171/MigrationUtil.java @@ -1,11 +1,17 @@ package org.openmetadata.service.migration.utils.v171; +import static org.openmetadata.service.Entity.ADMIN_USER_NAME; + import java.util.List; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; import org.openmetadata.schema.dataInsight.custom.LineChart; import org.openmetadata.schema.dataInsight.custom.LineChartMetric; +import org.openmetadata.schema.governance.workflows.WorkflowConfiguration; +import org.openmetadata.schema.governance.workflows.WorkflowDefinition; import org.openmetadata.service.jdbi3.DataInsightSystemChartRepository; +import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.jdbi3.WorkflowDefinitionRepository; import org.openmetadata.service.util.EntityUtil; @Slf4j @@ -14,6 +20,7 @@ public class MigrationUtil { private MigrationUtil() {} static DataInsightSystemChartRepository dataInsightSystemChartRepository; + static WorkflowDefinitionRepository workflowDefinitionRepository; public static void updateChart(String chartName, Object chartDetails) { DataInsightCustomChart chart = @@ -60,4 +67,26 @@ public class MigrationUtil { .withFormula("sum(k='tierSources.Generated')") .withName("ai")))); } + + public static void updateWorkflowDefinitions() { + workflowDefinitionRepository = new WorkflowDefinitionRepository(); + List workflowDefinitions = + workflowDefinitionRepository.listAll(EntityUtil.Fields.EMPTY_FIELDS, new ListFilter()); + + for (WorkflowDefinition workflowDefinition : workflowDefinitions) { + try { + if (workflowDefinition.getConfig() == null) { + workflowDefinition.setConfig(new WorkflowConfiguration().withStoreStageStatus(false)); + } else if (workflowDefinition.getConfig().getStoreStageStatus() == null) { + workflowDefinition.getConfig().setStoreStageStatus(false); + } + + workflowDefinitionRepository.createOrUpdate(null, workflowDefinition, ADMIN_USER_NAME); + } catch (Exception ex) { + LOG.warn(ex.toString()); + LOG.warn( + String.format("Error updating workflow definition %s", workflowDefinition.getName())); + } + } + } }