yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-01 13:13:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove Retry From Abstract Event Consumer (#16405)

Browse Source
This commit is contained in:
Mohit Yadav 2024-05-24 13:07:26 +05:30 committed by GitHub
parent 3a9c2473aa
commit f8ed079731
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 11 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/AbstractEventConsumer.java
View File

@ -16,17 +16,14 @@ package org.openmetadata.service.apps.bundles.changeEvent;
import static org.openmetadata.service.events.subscription.AlertUtil.getFilteredEvents; import static org.openmetadata.service.events.subscription.AlertUtil.getFilteredEvents;
import static org.openmetadata.service.events.subscription.AlertUtil.getStartingOffset; import static org.openmetadata.service.events.subscription.AlertUtil.getStartingOffset;
import com.fasterxml.jackson.core.type.TypeReference;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.UUID; import java.util.UUID;
import java.util.stream.Collectors;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -91,62 +88,17 @@ public abstract class AbstractEventConsumer
failingSubscriptionId, failingSubscriptionId,
changeEvent); changeEvent);
// Update Failed Event with details Entity.getCollectionDAO()
FailedEvent failedEvent = .eventSubscriptionDAO()
new FailedEvent() .upsertFailedEvent(
.withFailingSubscriptionId(failingSubscriptionId) eventSubscription.getId().toString(),
.withChangeEvent(changeEvent) String.format("%s-%s", FAILED_EVENT_EXTENSION, changeEvent.getId()),
.withRetriesLeft(eventSubscription.getRetries()) JsonUtils.pojoToJson(
.withTimestamp(System.currentTimeMillis()); new FailedEvent()
.withFailingSubscriptionId(failingSubscriptionId)
if (eventSubscription.getRetries() == 0) { .withChangeEvent(changeEvent)
Entity.getCollectionDAO() .withRetriesLeft(eventSubscription.getRetries())
.eventSubscriptionDAO() .withTimestamp(System.currentTimeMillis())));
.upsertFailedEvent(
eventSubscription.getId().toString(),
String.format("%s-%s", FAILED_EVENT_EXTENSION, changeEvent.getId()),
JsonUtils.pojoToJson(failedEvent));
} else {
// Check in Qtz Map
Set<FailedEvent> failedEventsList =
JsonUtils.convertValue(
jobDetail.getJobDataMap().get(FAILED_EVENT_EXTENSION), new TypeReference<>() {});
if (failedEventsList == null) {
failedEventsList = new HashSet<>();
} else {
// Remove exising change event
boolean removeChangeEvent =
failedEventsList.removeIf(
failedEvent1 -> {
if (failedEvent1
.getChangeEvent()
.getId()
.equals(failedEvent.getChangeEvent().getId())
&& failedEvent1.getFailingSubscriptionId().equals(failingSubscriptionId)) {
failedEvent.withRetriesLeft(failedEvent1.getRetriesLeft());
return true;
}
return false;
});
if (removeChangeEvent) {
if (failedEvent.getRetriesLeft() == 0) {
// If the Retries are exhausted, then remove the Event from the List to DLQ
failedEvent.withRetriesLeft(0);
} else {
failedEvent.withRetriesLeft(failedEvent.getRetriesLeft() - 1);
}
}
}
failedEventsList.add(failedEvent);
jobDetail.getJobDataMap().put(FAILED_EVENT_EXTENSION, failedEventsList);
Entity.getCollectionDAO()
.eventSubscriptionDAO()
.upsertFailedEvent(
eventSubscription.getId().toString(),
String.format("%s-%s", FAILED_EVENT_EXTENSION, changeEvent.getId()),
JsonUtils.pojoToJson(failedEvent));
}
} }
private long loadInitialOffset(JobExecutionContext context) { private long loadInitialOffset(JobExecutionContext context) {
@ -278,20 +230,6 @@ public abstract class AbstractEventConsumer
int batchSize = batch.size(); int batchSize = batch.size();
Map<ChangeEvent, Set<UUID>> eventsWithReceivers = createEventsWithReceivers(batch); Map<ChangeEvent, Set<UUID>> eventsWithReceivers = createEventsWithReceivers(batch);
try { try {
// Retry Failed Events
Set<FailedEvent> failedEventsList =
JsonUtils.convertValue(
jobDetail.getJobDataMap().get(FAILED_EVENT_EXTENSION), new TypeReference<>() {});
if (failedEventsList != null) {
Map<ChangeEvent, Set<UUID>> failedChangeEvents =
failedEventsList.stream()
.filter(failedEvent -> failedEvent.getRetriesLeft() > 0)
.collect(
Collectors.toMap(
FailedEvent::getChangeEvent,
failedEvent -> Set.of(failedEvent.getFailingSubscriptionId())));
eventsWithReceivers.putAll(failedChangeEvents);
}
// Publish Events // Publish Events
if (!eventsWithReceivers.isEmpty()) { if (!eventsWithReceivers.isEmpty()) {
alertMetrics.withTotalEvents(alertMetrics.getTotalEvents() + eventsWithReceivers.size()); alertMetrics.withTotalEvents(alertMetrics.getTotalEvents() + eventsWithReceivers.size());