diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/SettingsRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/SettingsRepository.java index bef510f3dff..a2b2c2b249f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/SettingsRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/SettingsRepository.java @@ -22,6 +22,7 @@ import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.openmetadata.catalog.filter.FilterRegistry; import org.openmetadata.catalog.filter.Filters; +import org.openmetadata.catalog.resources.settings.SettingsCache; import org.openmetadata.catalog.settings.Settings; import org.openmetadata.catalog.settings.SettingsType; import org.openmetadata.catalog.util.FilterUtil; @@ -118,6 +119,7 @@ public class SettingsRepository { .insertSettings(setting.getConfigType().toString(), JsonUtils.pojoToJson(setting.getConfigValue())); if (setting.getConfigType().equals(ACTIVITY_FEED_FILTER_SETTING)) { FilterRegistry.add(FilterUtil.getEventFilterFromSettings(setting)); + SettingsCache.getInstance().putSettings(setting); } } catch (Exception ex) { throw new RuntimeException(ex); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/kafka/KafkaWebhookEventPublisher.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/kafka/KafkaWebhookEventPublisher.java deleted file mode 100644 index 8c29495bd96..00000000000 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/kafka/KafkaWebhookEventPublisher.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.openmetadata.catalog.kafka; - -import com.fasterxml.jackson.core.JsonProcessingException; -import java.util.Properties; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.config.SslConfigs; -import org.openmetadata.catalog.events.WebhookPublisher; -import org.openmetadata.catalog.jdbi3.CollectionDAO; -import org.openmetadata.catalog.resources.events.EventResource; -import org.openmetadata.catalog.type.ChangeEvent; -import org.openmetadata.catalog.type.Webhook; -import org.openmetadata.catalog.util.JsonUtils; - -@Slf4j -public class KafkaWebhookEventPublisher extends WebhookPublisher { - - protected final Webhook webhook; - private static KafkaProducer producer; - Properties properties = new Properties(); - - public KafkaWebhookEventPublisher(Webhook webhook, CollectionDAO dao) { - super(webhook, dao); - this.webhook = webhook; - if (webhook.getKafkaProperties().getSecurityProtocol().equals(KafkaEventConfiguration.SecurityProtocol.SSL)) { - // configuration for SSL Encryption - if (webhook.getKafkaProperties().getSSLProtocol() != null - && webhook.getKafkaProperties().getSSLTrustStoreLocation() != null - && webhook.getKafkaProperties().getSSLTrustStorePassword() != null - && webhook.getKafkaProperties().getSSLKeystoreLocation() != null - && webhook.getKafkaProperties().getSSLKeystorePassword() != null - && webhook.getKafkaProperties().getSSLKeyPassword() != null) { - properties.put( - CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, webhook.getKafkaProperties().getSecurityProtocol()); - properties.put(SslConfigs.SSL_PROTOCOL_CONFIG, webhook.getKafkaProperties().getSSLProtocol()); - properties.put( - SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, webhook.getKafkaProperties().getSSLTrustStoreLocation()); - properties.put( - SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, webhook.getKafkaProperties().getSSLTrustStorePassword()); - // configuration for SSL Authentication - properties.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, webhook.getKafkaProperties().getSSLKeystoreLocation()); - properties.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, webhook.getKafkaProperties().getSSLKeystorePassword()); - properties.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, webhook.getKafkaProperties().getSSLKeyPassword()); - } else { - LOG.info("The SSL could not be configured as the required properties are not defined!"); - } - } - properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, webhook.getEndpoint().toString()); - properties.put(ProducerConfig.ACKS_CONFIG, webhook.getKafkaProperties().getAcks()); - properties.put(ProducerConfig.RETRIES_CONFIG, webhook.getKafkaProperties().getRetries()); - properties.put(ProducerConfig.BATCH_SIZE_CONFIG, webhook.getBatchSize()); - properties.put(ProducerConfig.LINGER_MS_CONFIG, webhook.getKafkaProperties().getLingerMS()); - properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, webhook.getKafkaProperties().getBufferMemory()); - properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, webhook.getKafkaProperties().getKeySerializer()); - properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, webhook.getKafkaProperties().getValueSerializer()); - producer = new KafkaProducer<>(properties); - } - - @Override - public void onStart() { - LOG.info("Kafka Webhook Publisher Started"); - } - - @Override - public void onShutdown() { - producer.flush(); - producer.close(); - } - - @Override - public void publish(EventResource.ChangeEventList events) throws JsonProcessingException { - if (webhook.getKafkaProperties().getTopics() != null) { - for (String topic : webhook.getKafkaProperties().getTopics()) { - for (ChangeEvent event : events.getData()) { - String eventJson = JsonUtils.pojoToJson(event); - producer.send(new ProducerRecord<>(topic, eventJson)); - } - } - } else { - LOG.info("Topics are null"); - } - } -} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java index 15f48f52be2..e059955f15e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java @@ -358,8 +358,7 @@ public class WebhookResource extends EntityResource .withTimeout(create.getTimeout()) .withEnabled(create.getEnabled()) .withSecretKey(create.getSecretKey()) - .withKafkaProperties(create.getKafkaProperties()) .withStatus(Boolean.TRUE.equals(create.getEnabled()) ? Status.ACTIVE : Status.DISABLED) - .withWebhookType(WebhookType.fromValue(create.getWebhookType().value())); + .withWebhookType(create.getWebhookType() == null ? WebhookType.generic : create.getWebhookType()); } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/settings/SettingsCache.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/settings/SettingsCache.java index d7cb0b73cc3..abf3e01e834 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/settings/SettingsCache.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/settings/SettingsCache.java @@ -61,6 +61,10 @@ public class SettingsCache { } } + public void putSettings(Settings setting) throws RuntimeException { + SETTINGS_CACHE.put(setting.getConfigType().toString(), setting); + } + public static void cleanUp() { SETTINGS_CACHE.invalidateAll(); INITIALIZED = false; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java index 4bada9a6d6b..22aeae55947 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java @@ -401,7 +401,10 @@ public final class EntityUtil { deleteFilter.ifPresent( eventFilter -> filters.add( - new Filters().withEventType(EventType.ENTITY_SOFT_DELETED).withFields(eventFilter.getFields()))); + new Filters() + .withEventType(EventType.ENTITY_SOFT_DELETED) + .withInclude(eventFilter.getInclude()) + .withExclude(eventFilter.getExclude()))); } public static EntityReference copy(EntityReference from, EntityReference to) { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/FilterUtil.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/FilterUtil.java index 16915c3492e..ca7eb8660b2 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/FilterUtil.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/FilterUtil.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.filter.EventFilter; import org.openmetadata.catalog.filter.Filters; import org.openmetadata.catalog.settings.Settings; @@ -36,6 +37,7 @@ import org.openmetadata.catalog.type.FieldChange; public class FilterUtil { private static final String TEST_CASE_RESULT = "testCaseResult"; + private static final String WILDCARD_FILTER = "all"; public static boolean shouldProcessRequest(ChangeEvent changeEvent, Map> filtersMap) { if (filtersMap != null && !filtersMap.isEmpty()) { @@ -44,20 +46,16 @@ public class FilterUtil { Map filtersOfEntity = filtersMap.get(entityType); if (filtersOfEntity == null || filtersOfEntity.size() == 0) { // check if we have all entities Filter - return handleWithWildCardFilter(filtersMap.get("all"), eventType, getUpdateField(changeEvent)); + return handleWithWildCardFilter(filtersMap.get(WILDCARD_FILTER), eventType, getUpdateField(changeEvent)); } else { Filters sf; if ((sf = filtersOfEntity.get(eventType)) == null) { return false; } else { - if (sf.getFields().contains("all")) { - return true; + if (entityType.equals(Entity.TEST_CASE)) { + return handleTestCaseFilter(changeEvent, sf); } else { - if (entityType.equals("testCase")) { - return handleTestCaseFilter(changeEvent, sf); - } else { - return checkIfFilterContainField(sf, getUpdateField(changeEvent)); - } + return checkIfFilterContainField(sf, getUpdateField(changeEvent)); } } } @@ -68,15 +66,18 @@ public class FilterUtil { private static boolean handleTestCaseFilter(ChangeEvent changeEvent, Filters sf) { List fieldChanges = getAllFieldChange(changeEvent); for (FieldChange fieldChange : fieldChanges) { - if (fieldChange.getName().equals(TEST_CASE_RESULT)) { + if (fieldChange.getName().equals(TEST_CASE_RESULT) && fieldChange.getNewValue() != null) { TestCaseResult testCaseResult = (TestCaseResult) fieldChange.getNewValue(); TestCaseStatus status = testCaseResult.getTestCaseStatus(); - if (sf.getFields().contains(TEST_CASE_RESULT + status.toString())) { + String statusField = TEST_CASE_RESULT + status.toString(); + if (sf.getInclude().contains(statusField)) { return true; + } else if (sf.getExclude().contains(statusField)) { + return false; } } } - return checkIfFilterContainField(sf, getUpdateField(changeEvent)); + return sf.getInclude().contains(WILDCARD_FILTER); } public static boolean handleWithWildCardFilter( @@ -84,8 +85,7 @@ public class FilterUtil { if (wildCardFilter != null && !wildCardFilter.isEmpty()) { // check if we have all entities Filter Filters f = wildCardFilter.get(type); - boolean allFieldCheck = checkIfFilterContainField(f, updatedField); - return f != null && (f.getFields().contains("all") || allFieldCheck); + return checkIfFilterContainField(f, updatedField); } return false; } @@ -93,10 +93,14 @@ public class FilterUtil { public static boolean checkIfFilterContainField(Filters f, List updatedField) { if (f != null) { for (String changed : updatedField) { - if (f.getFields().contains(changed)) { + // field is present in excluded + if (f.getExclude().contains(changed)) { + return false; + } else if (f.getInclude().contains(changed)) { return true; } } + return f.getInclude().contains(WILDCARD_FILTER); } return false; } @@ -111,12 +115,13 @@ public class FilterUtil { } public static List getAllFieldChange(ChangeEvent changeEvent) { - ChangeDescription description = changeEvent.getChangeDescription(); List allFieldChange = new ArrayList<>(); - allFieldChange.addAll(description.getFieldsAdded()); - allFieldChange.addAll(description.getFieldsUpdated()); - allFieldChange.addAll(description.getFieldsDeleted()); - + ChangeDescription description = changeEvent.getChangeDescription(); + if (description != null) { + allFieldChange.addAll(description.getFieldsAdded()); + allFieldChange.addAll(description.getFieldsUpdated()); + allFieldChange.addAll(description.getFieldsDeleted()); + } return allFieldChange; } diff --git a/catalog-rest-service/src/main/resources/json/data/settings/settingsData.json b/catalog-rest-service/src/main/resources/json/data/settings/settingsData.json index 00bf47fab08..c88e7a2a076 100644 --- a/catalog-rest-service/src/main/resources/json/data/settings/settingsData.json +++ b/catalog-rest-service/src/main/resources/json/data/settings/settingsData.json @@ -7,27 +7,210 @@ "filters": [ { "eventType": "entityCreated", - "fields": [ + "include": [ "all" - ] + ], + "exclude": [] }, { "eventType": "entityUpdated", - "fields": [ + "include": [ "all" - ] + ], + "exclude": ["usageSummary"] }, { "eventType": "entityDeleted", - "fields": [ + "include": [ "all" - ] + ], + "exclude": [] }, { "eventType": "entitySoftDeleted", - "fields": [ + "include": [ "all" - ] + ], + "exclude": [] + } + ] + }, + { + "entityType": "table", + "filters": [ + { + "eventType": "entityCreated", + "include": [ + "all" + ], + "exclude": [] + }, + { + "eventType": "entityUpdated", + "include": [ + "description", + "owner", + "tags", + "followers" + ], + "exclude": [] + }, + { + "eventType": "entityDeleted", + "include": [ + "all" + ], + "exclude": [] + }, + { + "eventType": "entitySoftDeleted", + "include": [ + "all" + ], + "exclude": [] + } + ] + }, + { + "entityType": "dashboard", + "filters": [ + { + "eventType": "entityCreated", + "include": [ + "all" + ], + "exclude": [] + }, + { + "eventType": "entityUpdated", + "include": [ + "description", + "owner", + "tags", + "followers" + ], + "exclude": [] + }, + { + "eventType": "entityDeleted", + "include": [ + "all" + ], + "exclude": [] + }, + { + "eventType": "entitySoftDeleted", + "include": [ + "all" + ], + "exclude": [] + } + ] + }, + { + "entityType": "pipeline", + "filters": [ + { + "eventType": "entityCreated", + "include": [ + "all" + ], + "exclude": [] + }, + { + "eventType": "entityUpdated", + "include": [ + "description", + "owner", + "tags", + "followers" + ], + "exclude": [] + }, + { + "eventType": "entityDeleted", + "include": [ + "all" + ], + "exclude": [] + }, + { + "eventType": "entitySoftDeleted", + "include": [ + "all" + ], + "exclude": [] + } + ] + }, + { + "entityType": "mlmodel", + "filters": [ + { + "eventType": "entityCreated", + "include": [ + "all" + ], + "exclude": [] + }, + { + "eventType": "entityUpdated", + "include": [ + "description", + "owner", + "tags", + "followers" + ], + "exclude": [] + }, + { + "eventType": "entityDeleted", + "include": [ + "all" + ], + "exclude": [] + }, + { + "eventType": "entitySoftDeleted", + "include": [ + "all" + ], + "exclude": [] + } + ] + }, + { + "entityType": "testCase", + "filters": [ + { + "eventType": "entityCreated", + "include": [ + "all" + ], + "exclude": [] + }, + { + "eventType": "entityUpdated", + "include": [ + "testCaseResultSuccess", + "testCaseResultFailed", + "testCaseResultAborted" + ], + "exclude": [] + }, + { + "eventType": "entityDeleted", + "include": [ + "all" + ], + "exclude": [] + }, + { + "eventType": "entitySoftDeleted", + "include": [ + "all" + ], + "exclude": [] } ] } diff --git a/catalog-rest-service/src/main/resources/json/schema/api/events/createWebhook.json b/catalog-rest-service/src/main/resources/json/schema/api/events/createWebhook.json index 64d67201444..588523e9764 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/events/createWebhook.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/events/createWebhook.json @@ -51,12 +51,8 @@ "description": "Secret set by the webhook client used for computing HMAC SHA256 signature of webhook payload and sent in `X-OM-Signature` header in POST requests to publish the events.", "type": "string" }, - "kafkaProperties": { - "description": "Properties of Kafka Producer", - "$ref": "../../configuration/kafkaEventConfiguration.json" - }, "webhookType": { - "description": "Type of webhook slack,generic,kafka etc", + "description": "Type of webhook slack,generic,msteams etc", "$ref": "../../entity/events/webhook.json#/definitions/webhookType" } }, diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json b/catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json index 387069dbc3d..aeec59635c2 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json @@ -8,7 +8,7 @@ "javaInterfaces": ["org.openmetadata.catalog.EntityInterface"], "definitions": { "webhookType": { - "description": "Type of webhook slack, generic, kafka, etc.", + "description": "Type of webhook slack, generic, msteams, etc.", "type": "string", "javaType": "org.openmetadata.catalog.type.WebhookType", "default": "generic", @@ -44,7 +44,7 @@ "type": "string" }, "webhookType": { - "description": "Type of webhook slack, generic, kafka, etc.", + "description": "Type of webhook slack, generic, msteams, etc.", "$ref": "#/definitions/webhookType" }, "description": { @@ -68,10 +68,6 @@ "type": "integer", "default": 10 }, - "kafkaProperties": { - "description": "Properties of Kafka Producer", - "$ref": "../../configuration/kafkaEventConfiguration.json" - }, "timeout": { "description": "Connection timeout in seconds. (Default 10s).", "type": "integer", diff --git a/catalog-rest-service/src/main/resources/json/schema/type/changeEvent.json b/catalog-rest-service/src/main/resources/json/schema/type/changeEvent.json index c40fb84722b..f4d5af58616 100644 --- a/catalog-rest-service/src/main/resources/json/schema/type/changeEvent.json +++ b/catalog-rest-service/src/main/resources/json/schema/type/changeEvent.json @@ -17,6 +17,140 @@ "entityDeleted" ] }, + "entityTypes": { + "javaType": "org.openmetadata.catalog.filter.EntityTypes", + "description": "Type of event.", + "type": "string", + "enum": [ + "All", + "Chart", + "Dashboard", + "Database", + "Database Schema", + "Glossary", + "Glossary Term", + "Location", + "Metrics", + "Ml Model", + "Pipeline", + "Report", + "Table", + "Topic", + "Test Case" + ], + "javaEnums": [ + { + "name": "all" + }, + { + "name": "chart" + }, + { + "name": "dashboard" + }, + { + "name": "database" + }, + { + "name": "databaseSchema" + }, + { + "name": "glossary" + }, + { + "name": "glossaryTerm" + }, + { + "name": "location" + }, + { + "name": "metrics" + }, + { + "name": "mlmodel" + }, + { + "name": "pipeline" + }, + { + "name": "report" + }, + { + "name": "table" + }, + { + "name": "topic" + }, + { + "name": "testCase" + } + ] + }, + "fieldTypes": { + "javaType": "org.openmetadata.catalog.filter.FieldType", + "description": "Type of event.", + "type": "string", + "enum": [ + "All", + "Display Name", + "Description", + "Owner", + "Location", + "Tags", + "Usage Summary", + "Followers", + "Sample Data", + "Synonyms", + "Glossary", + "Test Case Result Success", + "Test Case Result Failed", + "Test Case Result Aborted" + ], + "javaEnums": [ + { + "name": "all" + }, + { + "name": "displayName" + }, + { + "name": "description" + }, + { + "name": "owner" + }, + { + "name": "location" + }, + { + "name": "tags" + }, + { + "name": "usageSummary" + }, + { + "name": "followers" + }, + { + "name": "sampleData" + }, + { + "name": "synonyms" + }, + { + "name": "glossary" + }, + { + "name": "testCaseResultSuccess" + }, + { + "name": "testCaseResultFailure" + }, + { + "name": "testCaseResultAborted" + } + ] + }, "filters": { "type": "object", "javaType": "org.openmetadata.catalog.filter.Filters", @@ -25,14 +159,23 @@ "description": "Event type that is being requested.", "$ref": "#/definitions/eventType" }, - "fields": { - "description": "Field on which to apply the filter on", + "include": { + "description": "Field which are allowed to pass", "type": "array", "items": { "type": "string" }, "default": ["all"], "uniqueItems": true + }, + "exclude": { + "description": "Field which are not allowed to pass", + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "uniqueItems": true } }, "required": ["eventType"], diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java index 3d15ce84db7..3d71672f22e 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java @@ -22,7 +22,6 @@ import static org.openmetadata.catalog.util.EntityUtil.fieldDeleted; import static org.openmetadata.catalog.util.EntityUtil.fieldUpdated; import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import java.io.IOException; import java.net.URI; @@ -65,16 +64,14 @@ public class WebhookResourceTest extends EntityResourceTest()), + new Filters().withEventType(EventType.ENTITY_UPDATED).withInclude(allFilter).withExclude(new HashSet<>()), + new Filters().withEventType(EventType.ENTITY_DELETED).withInclude(allFilter).withExclude(new HashSet<>()), + new Filters() + .withEventType(EventType.ENTITY_SOFT_DELETED) + .withInclude(allFilter) + .withExclude(new HashSet<>()))); ALL_EVENTS_FILTER.add(allEntityFilter); - try { - System.out.println(JsonUtils.pojoToJson(ALL_EVENTS_FILTER)); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } } public WebhookResourceTest() { @@ -189,9 +186,12 @@ public class WebhookResourceTest extends EntityResourceTest allFilter = new HashSet<>(); allFilter.add("all"); - Filters createFilter = new Filters().withEventType(EventType.ENTITY_CREATED).withFields(allFilter); - Filters updateFilter = new Filters().withEventType(EventType.ENTITY_UPDATED).withFields(allFilter); - Filters deleteFilter = new Filters().withEventType(EventType.ENTITY_DELETED).withFields(allFilter); + Filters createFilter = + new Filters().withEventType(EventType.ENTITY_CREATED).withInclude(allFilter).withExclude(new HashSet<>()); + Filters updateFilter = + new Filters().withEventType(EventType.ENTITY_UPDATED).withInclude(allFilter).withExclude(new HashSet<>()); + Filters deleteFilter = + new Filters().withEventType(EventType.ENTITY_DELETED).withInclude(allFilter).withExclude(new HashSet<>()); EventFilter f1 = new EventFilter().withEntityType("all").withFilters(List.of(createFilter)); EventFilter f2 = @@ -300,16 +300,18 @@ public class WebhookResourceTest extends EntityResourceTest allFiler = new HashSet<>(); - allFiler.add("all"); - Filters createFilter = new Filters().withEventType(EventType.ENTITY_CREATED).withFields(allFiler); + Set allFilter = new HashSet<>(); + allFilter.add("all"); + Filters createFilter = + new Filters().withEventType(EventType.ENTITY_CREATED).withInclude(allFilter).withExclude(new HashSet<>()); EventFilter f1 = new EventFilter().withEntityType(entity).withFilters(List.of(createFilter)); createWebhook(name, uri, List.of(f1)); // Create webhook with endpoint api/v1/test/webhook/entityUpdated/ to receive entityUpdated events name = EventType.ENTITY_UPDATED + ":" + entity; uri = baseUri + "/" + EventType.ENTITY_UPDATED + "/" + entity; - Filters updateFilter = new Filters().withEventType(EventType.ENTITY_UPDATED).withFields(allFiler); + Filters updateFilter = + new Filters().withEventType(EventType.ENTITY_UPDATED).withInclude(allFilter).withExclude(new HashSet<>()); EventFilter f2 = new EventFilter().withEntityType(entity).withFilters(List.of(updateFilter)); createWebhook(name, uri, List.of(f2)); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddWebhook/AddWebhook.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddWebhook/AddWebhook.test.tsx index 0df1e7e575c..4bdeff4c106 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddWebhook/AddWebhook.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddWebhook/AddWebhook.test.tsx @@ -90,19 +90,23 @@ const mockData = { filters: [ { eventType: 'entityCreated', - fields: ['all'], + include: ['all'], + exclude: [], }, { eventType: 'entityUpdated', - fields: ['all'], + include: ['all'], + exclude: [], }, { eventType: 'entityDeleted', - fields: ['all'], + include: ['all'], + exclude: [], }, { eventType: 'entitySoftDeleted', - fields: ['all'], + include: ['all'], + exclude: [], }, ], }, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddWebhook/EventFilterSelect.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddWebhook/EventFilterSelect.component.tsx index 647aa5b4bbd..31fd48115b0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddWebhook/EventFilterSelect.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddWebhook/EventFilterSelect.component.tsx @@ -38,8 +38,8 @@ const EventFilterSelect = ({ metric === EventType.EntityUpdated ? Object.values(EventUpdateTypes).map((updateType) => ({ title: startCase(updateType), - value: updateType, - key: updateType, + value: `${EventType.EntityUpdated}-${updateType}`, + key: `${EventType.EntityUpdated}-${updateType}`, })) : undefined, })), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddWebhook/WebhookConstants.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddWebhook/WebhookConstants.ts index 69d4d5f7713..477a48f76d7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddWebhook/WebhookConstants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddWebhook/WebhookConstants.ts @@ -34,19 +34,23 @@ export const EVENT_FILTERS_DEFAULT_VALUE = { filters: [ { eventType: 'entityCreated', - fields: ['all'], + include: ['all'], + exclude: [], }, { eventType: 'entityUpdated', - fields: ['all'], + include: ['all'], + exclude: [], }, { eventType: 'entityDeleted', - fields: ['all'], + include: ['all'], + exclude: [], }, { eventType: 'entitySoftDeleted', - fields: ['all'], + include: ['all'], + exclude: [], }, ], } as EventFilter; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ActivityFeedSettingsPage/ActivityFeedSettingsPage.constants.ts b/openmetadata-ui/src/main/resources/ui/src/pages/ActivityFeedSettingsPage/ActivityFeedSettingsPage.constants.ts index 8bb61e10e2f..34faee00b15 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ActivityFeedSettingsPage/ActivityFeedSettingsPage.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ActivityFeedSettingsPage/ActivityFeedSettingsPage.constants.ts @@ -3,19 +3,23 @@ import { Filters } from '../../generated/settings/settings'; export const formData = [ { eventType: 'entityCreated', - fields: [], + include: [], + exclude: [], }, { eventType: 'entityUpdated', - fields: [], + include: [], + exclude: [], }, { eventType: 'entitySoftDeleted', - fields: [], + include: [], + exclude: [], }, { eventType: 'entityDeleted', - fields: [], + include: [], + exclude: [], }, ] as Filters[]; @@ -25,4 +29,5 @@ export const ActivityFeedEntity = { dashboard: 'Dashboard', pipeline: 'Pipeline', mlmodel: 'ML Model', + testCase: 'Test Case', } as Record; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ActivityFeedSettingsPage/ActivityFeedSettingsPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ActivityFeedSettingsPage/ActivityFeedSettingsPage.tsx index 2fea4ed429d..7c3a2e0cf94 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ActivityFeedSettingsPage/ActivityFeedSettingsPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ActivityFeedSettingsPage/ActivityFeedSettingsPage.tsx @@ -93,13 +93,13 @@ const ActivityFeedSettingsPage: React.FC = () => { }; const generateTreeData = (entityType: string, data?: Filters[]) => { - return data?.map(({ eventType, fields }) => { + return data?.map(({ eventType, include }) => { const key = `${entityType}-${eventType}` as string; return { key: key, title: startCase(eventType), - data: fields, + data: include, children: eventType === 'entityUpdated' ? [ @@ -133,13 +133,13 @@ const ActivityFeedSettingsPage: React.FC = () => { filters && filters.map((obj) => { if ( - obj?.fields && - obj.fields.length === 1 && - obj.fields[0] === 'all' + obj.include && + obj.include.length === 1 && + obj.include[0] === 'all' ) { checkedArray.push(`${entityType}-${obj.eventType}`); } else { - obj?.fields?.forEach((entityUpdated) => { + obj?.include?.forEach((entityUpdated) => { const name = `${entityType}-${obj.eventType}-${entityUpdated}`; checkedArray.push(name); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ActivityFeedSettingsPage/ActivityFeedSettingsPage.utils.ts b/openmetadata-ui/src/main/resources/ui/src/pages/ActivityFeedSettingsPage/ActivityFeedSettingsPage.utils.ts index 9071734c958..526fc710a9f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ActivityFeedSettingsPage/ActivityFeedSettingsPage.utils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ActivityFeedSettingsPage/ActivityFeedSettingsPage.utils.ts @@ -1,5 +1,8 @@ import { isEmpty, isUndefined } from 'lodash'; import { Filters } from '../../generated/settings/settings'; +import { getDiffArray } from '../../utils/CommonUtils'; + +const entityUpdatedFields = ['description', 'owner', 'tags', 'followers']; export const getPayloadFromSelected = ( selectedOptions: Record, @@ -25,7 +28,8 @@ export const getPayloadFromSelected = ( ...valueAcc, { eventType: selected[1], - fields: ['all'], + include: ['all'], + exclude: [], }, ]; } else { @@ -39,11 +43,14 @@ export const getPayloadFromSelected = ( resultArr.push(...arr[0]); if (!isUndefined(nonUpdatedFields) && !isEmpty(nonUpdatedFields)) { + const selectedUpdatedData = nonUpdatedFields.filter( + (name) => !isUndefined(name) || (!isEmpty(name) && name) + ); + resultArr.push({ eventType: 'entityUpdated', - fields: nonUpdatedFields.filter( - (name) => !isUndefined(name) || (!isEmpty(name) && name) - ), + include: selectedUpdatedData, + exclude: getDiffArray(entityUpdatedFields, selectedUpdatedData), }); } diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx index 25e2bbc4be9..727976af384 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx @@ -14,7 +14,15 @@ import { Popover } from 'antd'; import { AxiosError } from 'axios'; import classNames from 'classnames'; -import { capitalize, isEmpty, isNil, isNull, isUndefined } from 'lodash'; +import { + capitalize, + differenceWith, + isEmpty, + isEqual, + isNil, + isNull, + isUndefined, +} from 'lodash'; import { EntityFieldThreadCount, ExtraInfo, @@ -879,3 +887,10 @@ export const getIngestionStatuses = (ingestion: IngestionPipeline) => { ); }); }; + +export const getDiffArray = ( + compareWith: string[], + toCompare: string[] +): string[] => { + return differenceWith(compareWith, toCompare, isEqual); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/WebhookUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/WebhookUtils.ts index 69a4dd21616..df9f39ad047 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/WebhookUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/WebhookUtils.ts @@ -1,11 +1,15 @@ import { Store } from 'antd/lib/form/interface'; -import { isEqual } from 'lodash'; +import { isEqual, isUndefined } from 'lodash'; import { EVENT_FILTERS_DEFAULT_VALUE, EVENT_FILTER_FORM_INITIAL_VALUE, } from '../components/AddWebhook/WebhookConstants'; import { TERM_ALL } from '../constants/constants'; -import { EventFilter, Filters } from '../generated/entity/events/webhook'; +import { + EventFilter, + EventType, + Filters, +} from '../generated/entity/events/webhook'; export const getEventFilters = (data: Store): EventFilter[] => { if (isEqual(data, EVENT_FILTER_FORM_INITIAL_VALUE)) { @@ -18,17 +22,39 @@ export const getEventFilters = (data: Store): EventFilter[] => { } if (value) { const selectedFilter = data[`${key}-tree`] as string[]; + const entityUpdatedFields = selectedFilter + ?.map((filter) => + filter.includes(`${EventType.EntityUpdated}-`) + ? filter.split('-')[1] + : undefined + ) + .filter((filter) => filter); + + const eventFilters = [ + ...selectedFilter.filter( + (filter) => !filter.includes(`${EventType.EntityUpdated}-`) + ), + entityUpdatedFields ? EventType.EntityUpdated : undefined, + ]; + + const includeData = (entityUpdatedFields as string[]).filter( + (entityUpdatedField) => !isUndefined(entityUpdatedField) + ); return [ ...acc, { entityType: key, filters: - selectedFilter[0] === TERM_ALL + eventFilters[0] === TERM_ALL ? EVENT_FILTERS_DEFAULT_VALUE.filters - : (selectedFilter.map((filter) => ({ + : (eventFilters.map((filter) => ({ eventType: filter, - fields: [TERM_ALL], + include: + filter === EventType.EntityUpdated + ? includeData + : [TERM_ALL], + exclude: [], })) as Filters[]), }, ];