diff --git a/ingestion/src/metadata/workflow/context/context_manager.py b/ingestion/src/metadata/workflow/context/context_manager.py index 9b8632db5e2..dcc1e996674 100644 --- a/ingestion/src/metadata/workflow/context/context_manager.py +++ b/ingestion/src/metadata/workflow/context/context_manager.py @@ -116,7 +116,9 @@ class ContextManager: result: dict[str, Any] = {} for context_enum in ContextsEnum: context_obj = getattr(instance, context_enum.value) - result[context_enum.value] = context_obj.model_dump() + context_dict = context_obj.model_dump(exclude_none=True) + if context_dict: + result[context_enum.value] = context_dict if result: return result return None diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java index 581aef584e1..62da0eef7a5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java @@ -19,6 +19,7 @@ import static org.openmetadata.schema.type.EventType.ENTITY_UPDATED; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriInfo; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; import lombok.Getter; @@ -299,6 +300,29 @@ public class IngestionPipelineRepository extends EntityRepository listExternalAppStatus( + String ingestionPipelineFQN, String serviceName, Long startTs, Long endTs) { + return listPipelineStatus(ingestionPipelineFQN, startTs, endTs) + .filter( + pipelineStatus -> { + Map metadata = pipelineStatus.getMetadata(); + if (metadata == null) { + return false; + } + Map workflowMetadata = + JsonUtils.readOrConvertValue(metadata.get("workflow"), Map.class); + String pipelineStatusService = (String) workflowMetadata.get("serviceName"); + return pipelineStatusService != null && pipelineStatusService.equals(serviceName); + }) + .map( + pipelineStatus -> + pipelineStatus.withConfig( + Optional.ofNullable(pipelineStatus.getConfig()) + .map(m -> m.getOrDefault("appConfig", null)) + .map(JsonUtils::getMap) + .orElse(null))); + } + public PipelineStatus getLatestPipelineStatus(IngestionPipeline ingestionPipeline) { return JsonUtils.readValue( getLatestExtensionFromTimeSeries( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java index 1a39aaf02f3..d7352242230 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import jakarta.validation.constraints.NotNull; import java.util.List; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import org.openmetadata.schema.system.EntityError; import org.openmetadata.schema.type.Paging; @@ -102,6 +103,11 @@ public class ResultList { return new ResultList<>(data.stream().map(mapper).collect(Collectors.toList()), paging); } + /* Conveniently filter the data without the need to create a new ResultList */ + public ResultList filter(Predicate predicate) { + return new ResultList<>(data.stream().filter(predicate).collect(Collectors.toList()), paging); + } + public ResultList(List data, Integer offset, Integer limit, Integer total) { this.data = data; paging =