yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-27 18:36:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Workflow improvements (#20054)

Browse Source
This commit is contained in:
IceS2 2025-03-04 14:08:45 +01:00 committed by GitHub
parent 6f0dbd01eb
commit 70a2aa1778
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 26 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CreateIngestionPipelineTask.java
View File

@ -30,7 +30,7 @@ public class CreateIngestionPipelineTask implements NodeInterface {
String subProcessId = nodeDefinition.getName(); String subProcessId = nodeDefinition.getName();
SubProcess subProcess = SubProcess subProcess =
new SubProcessBuilder().id(subProcessId).setAsync(true).exclusive(false).build(); new SubProcessBuilder().id(subProcessId).setAsync(true).exclusive(true).build();
StartEvent startEvent = StartEvent startEvent =
new StartEventBuilder().id(getFlowableElementId(subProcessId, "startEvent")).build(); new StartEventBuilder().id(getFlowableElementId(subProcessId, "startEvent")).build();

14
openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/CreateIngestionPipelineImpl.java
View File

@ -146,15 +146,27 @@ public class CreateIngestionPipelineImpl implements JavaDelegate {
private IngestionPipeline createIngestionPipeline( private IngestionPipeline createIngestionPipeline(
IngestionPipelineMapper mapper, PipelineType pipelineType, ServiceEntityInterface service) { IngestionPipelineMapper mapper, PipelineType pipelineType, ServiceEntityInterface service) {
String displayName = String.format("[%s] %s", service.getName(), pipelineType);
IngestionPipelineRepository repository = IngestionPipelineRepository repository =
(IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE); (IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE);
for (String ingestionPipelineStr :
repository.listAllByParentFqn(service.getFullyQualifiedName())) {
IngestionPipeline ingestionPipeline =
JsonUtils.readOrConvertValue(ingestionPipelineStr, IngestionPipeline.class);
if (ingestionPipeline.getPipelineType().equals(pipelineType)
&& ingestionPipeline.getDisplayName().equals(displayName)) {
return ingestionPipeline;
}
}
;
org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline create = org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline create =
new org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline() new org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline()
.withAirflowConfig(new AirflowConfig().withStartDate(getYesterdayDate())) .withAirflowConfig(new AirflowConfig().withStartDate(getYesterdayDate()))
.withLoggerLevel(LogLevels.INFO) .withLoggerLevel(LogLevels.INFO)
.withName(UUID.randomUUID().toString()) .withName(UUID.randomUUID().toString())
.withDisplayName(String.format("[%s] %s", service.getName(), pipelineType)) .withDisplayName(displayName)
.withOwners(service.getOwners()) .withOwners(service.getOwners())
.withPipelineType(pipelineType) .withPipelineType(pipelineType)
.withService(service.getEntityReference()) .withService(service.getEntityReference())

13
openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppImpl.java
View File

@ -7,6 +7,7 @@ import java.util.List;
import java.util.Set; import java.util.Set;
import javax.json.JsonPatch; import javax.json.JsonPatch;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.ServiceEntityInterface; import org.openmetadata.schema.ServiceEntityInterface;
import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.App;
import org.openmetadata.schema.entity.app.AppRunRecord; import org.openmetadata.schema.entity.app.AppRunRecord;
@ -29,6 +30,7 @@ import org.openmetadata.sdk.PipelineServiceClientInterface;
import org.openmetadata.service.Entity; import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.apps.ApplicationHandler; import org.openmetadata.service.apps.ApplicationHandler;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.AppRepository;
import org.openmetadata.service.jdbi3.IngestionPipelineRepository; import org.openmetadata.service.jdbi3.IngestionPipelineRepository;
import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.resources.feeds.MessageParser;
@ -36,6 +38,7 @@ import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.OpenMetadataConnectionBuilder; import org.openmetadata.service.util.OpenMetadataConnectionBuilder;
@Slf4j
public class RunAppImpl { public class RunAppImpl {
public boolean execute( public boolean execute(
PipelineServiceClientInterface pipelineServiceClient, PipelineServiceClientInterface pipelineServiceClient,
@ -46,8 +49,14 @@ public class RunAppImpl {
ServiceEntityInterface service = Entity.getEntity(entityLink, "owners", Include.NON_DELETED); ServiceEntityInterface service = Entity.getEntity(entityLink, "owners", Include.NON_DELETED);
AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION);
App app = App app;
appRepository.getByName(null, appName, new EntityUtil.Fields(Set.of("bot", "pipelines"))); try {
app =
appRepository.getByName(null, appName, new EntityUtil.Fields(Set.of("bot", "pipelines")));
} catch (EntityNotFoundException ex) {
LOG.warn(String.format("App: '%s' is not Installed. Skipping", appName));
return true;
}
if (!validateAppShouldRun(app, service)) { if (!validateAppShouldRun(app, service)) {
return true; return true;

2
openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppTask.java
View File

@ -28,7 +28,7 @@ public class RunAppTask implements NodeInterface {
String subProcessId = nodeDefinition.getName(); String subProcessId = nodeDefinition.getName();
SubProcess subProcess = SubProcess subProcess =
new SubProcessBuilder().id(subProcessId).setAsync(true).exclusive(false).build(); new SubProcessBuilder().id(subProcessId).setAsync(true).exclusive(true).build();
StartEvent startEvent = StartEvent startEvent =
new StartEventBuilder().id(getFlowableElementId(subProcessId, "startEvent")).build(); new StartEventBuilder().id(getFlowableElementId(subProcessId, "startEvent")).build();