MINOR: Remove workflow instance states (#18959)

* Remove Workflow Instance States until better solution is found

* Remove Workflow Instance States until better solution is found

* Remove Workflow Instance States until better solution is found

* Fix issue with OpenSearchClient

openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/MainWorkflowTerminationListener.java

@@ -1,6 +1,5 @@
 package org.openmetadata.service.governance.workflows;
 
-import static org.openmetadata.service.governance.workflows.Workflow.STAGE_INSTANCE_STATE_ID_VARIABLE;
 import static org.openmetadata.service.governance.workflows.WorkflowHandler.getProcessDefinitionKeyFromId;
 
 import java.util.UUID;
@@ -9,21 +8,12 @@ import org.flowable.engine.delegate.DelegateExecution;
 import org.flowable.engine.delegate.JavaDelegate;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.jdbi3.WorkflowInstanceRepository;
-import org.openmetadata.service.jdbi3.WorkflowInstanceStateRepository;
 
 @Slf4j
 public class MainWorkflowTerminationListener implements JavaDelegate {
   @Override
   public void execute(DelegateExecution execution) {
     try {
-      WorkflowInstanceStateRepository workflowInstanceStateRepository =
-          (WorkflowInstanceStateRepository)
-              Entity.getEntityTimeSeriesRepository(Entity.WORKFLOW_INSTANCE_STATE);
-
-      UUID workflowInstanceStateId = (UUID) execution.getVariable(STAGE_INSTANCE_STATE_ID_VARIABLE);
-      workflowInstanceStateRepository.updateStage(
-          workflowInstanceStateId, System.currentTimeMillis(), execution.getVariables());
-
       WorkflowInstanceRepository workflowInstanceRepository =
           (WorkflowInstanceRepository)
               Entity.getEntityTimeSeriesRepository(Entity.WORKFLOW_INSTANCE);

openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CheckEntityAttributesTask.java

@@ -45,8 +45,6 @@ public class CheckEntityAttributesTask implements NodeInterface {
     subProcess.addFlowElement(new SequenceFlow(startEvent.getId(), checkEntityAttributes.getId()));
     subProcess.addFlowElement(new SequenceFlow(checkEntityAttributes.getId(), endEvent.getId()));
 
-    attachWorkflowInstanceStageListeners(subProcess);
-
     this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess);
     this.subProcess = subProcess;
   }

openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetEntityCertificationTask.java

@@ -50,8 +50,6 @@ public class SetEntityCertificationTask implements NodeInterface {
     subProcess.addFlowElement(new SequenceFlow(startEvent.getId(), setEntityCertification.getId()));
     subProcess.addFlowElement(new SequenceFlow(setEntityCertification.getId(), endEvent.getId()));
 
-    attachWorkflowInstanceStageListeners(subProcess);
-
     this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess);
     this.subProcess = subProcess;
   }

openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetGlossaryTermStatusTask.java

@@ -46,8 +46,6 @@ public class SetGlossaryTermStatusTask implements NodeInterface {
     subProcess.addFlowElement(new SequenceFlow(startEvent.getId(), setGlossaryTermStatus.getId()));
     subProcess.addFlowElement(new SequenceFlow(setGlossaryTermStatus.getId(), endEvent.getId()));
 
-    attachWorkflowInstanceStageListeners(subProcess);
-
     this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess);
     this.subProcess = subProcess;
   }

openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/endEvent/EndEvent.java

@@ -13,12 +13,10 @@ public class EndEvent implements NodeInterface {
 
   public EndEvent(String id) {
     this.endEvent = new EndEventBuilder().id(id).build();
-    attachWorkflowInstanceStageListeners(endEvent);
   }
 
   public EndEvent(EndEventDefinition nodeDefinition) {
     this.endEvent = new EndEventBuilder().id(nodeDefinition.getName()).build();
-    attachWorkflowInstanceStageListeners(endEvent);
   }
 
   public void addToWorkflow(BpmnModel model, Process process) {

openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/startEvent/StartEvent.java

@@ -12,7 +12,6 @@ public class StartEvent implements NodeInterface {
   public StartEvent(StartEventDefinition nodeDefinition) {
     this.startEvent = new StartEventBuilder().id(nodeDefinition.getName()).build();
     attachWorkflowInstanceExecutionIdSetterListener(startEvent);
-    attachWorkflowInstanceStageListeners(startEvent);
   }
 
   public void addToWorkflow(BpmnModel model, Process process) {

openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/UserApprovalTask.java

@@ -91,8 +91,6 @@ public class UserApprovalTask implements NodeInterface {
     subProcess.addFlowElement(new SequenceFlow(userTask.getId(), endEvent.getId()));
     subProcess.addFlowElement(new SequenceFlow(terminationEvent.getId(), terminatedEvent.getId()));
 
-    attachWorkflowInstanceStageListeners(subProcess);
-
     this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess);
     this.subProcess = subProcess;
   }

openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/CreateApprovalTaskImpl.java

@@ -2,7 +2,6 @@ package org.openmetadata.service.governance.workflows.elements.nodes.userTask.im
 
 import static org.openmetadata.service.governance.workflows.Workflow.EXCEPTION_VARIABLE;
 import static org.openmetadata.service.governance.workflows.Workflow.RELATED_ENTITY_VARIABLE;
-import static org.openmetadata.service.governance.workflows.Workflow.STAGE_INSTANCE_STATE_ID_VARIABLE;
 import static org.openmetadata.service.governance.workflows.Workflow.WORKFLOW_RUNTIME_EXCEPTION;
 import static org.openmetadata.service.governance.workflows.WorkflowHandler.getProcessDefinitionKeyFromId;
 
@@ -27,7 +26,6 @@ import org.openmetadata.service.Entity;
 import org.openmetadata.service.exception.EntityNotFoundException;
 import org.openmetadata.service.governance.workflows.WorkflowHandler;
 import org.openmetadata.service.jdbi3.FeedRepository;
-import org.openmetadata.service.jdbi3.WorkflowInstanceStateRepository;
 import org.openmetadata.service.resources.feeds.FeedResource;
 import org.openmetadata.service.resources.feeds.MessageParser;
 import org.openmetadata.service.util.WebsocketNotificationHandler;
@@ -45,13 +43,6 @@ public class CreateApprovalTaskImpl implements TaskListener {
 
       Thread task = createApprovalTask(entity, assignees);
       WorkflowHandler.getInstance().setCustomTaskId(delegateTask.getId(), task.getId());
-
-      UUID workflowInstanceStateId =
-          (UUID) delegateTask.getVariable(STAGE_INSTANCE_STATE_ID_VARIABLE);
-      WorkflowInstanceStateRepository workflowInstanceStateRepository =
-          (WorkflowInstanceStateRepository)
-              Entity.getEntityTimeSeriesRepository(Entity.WORKFLOW_INSTANCE_STATE);
-      workflowInstanceStateRepository.updateStageWithTask(task.getId(), workflowInstanceStateId);
     } catch (Exception exc) {
       LOG.error(
           String.format(

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowInstanceRepository.java

@@ -3,6 +3,7 @@ package org.openmetadata.service.jdbi3;
 import static org.openmetadata.service.governance.workflows.Workflow.EXCEPTION_VARIABLE;
 
 import java.util.Map;
+import java.util.Optional;
 import java.util.UUID;
 import org.openmetadata.schema.governance.workflows.WorkflowInstance;
 import org.openmetadata.service.Entity;
@@ -50,7 +51,7 @@ public class WorkflowInstanceRepository extends EntityTimeSeriesRepository<Workf
 
     workflowInstance.setEndedAt(endedAt);
 
-    if (variables.containsKey(EXCEPTION_VARIABLE)) {
+    if (Optional.ofNullable(variables.getOrDefault(EXCEPTION_VARIABLE, null)).isPresent()) {
       workflowInstance.setException(true);
     }
 

openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java

@@ -737,9 +737,16 @@ public class OpenSearchClient implements SearchClient {
               new os.org.opensearch.action.search.SearchRequest(index).source(searchSourceBuilder),
               RequestOptions.DEFAULT);
       SearchHits searchHits = response.getHits();
-      SearchHit[] hits = searchHits.getHits();
+      List<SearchHit> hits = List.of(searchHits.getHits());
-      Arrays.stream(hits).forEach(hit -> results.add(hit.getSourceAsMap()));
+      Object[] lastHitSortValues = null;
-      return new SearchResultListMapper(results, searchHits.getTotalHits().value);
+
+      if (!hits.isEmpty()) {
+        lastHitSortValues = hits.get(hits.size() - 1).getSortValues();
+      }
+
+      hits.forEach(hit -> results.add(hit.getSourceAsMap()));
+      return new SearchResultListMapper(
+          results, searchHits.getTotalHits().value, lastHitSortValues);
     } catch (OpenSearchStatusException e) {
       if (e.status() == RestStatus.NOT_FOUND) {
         throw new SearchIndexNotFoundException(String.format("Failed to to find index %s", index));