diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchEventPublisher.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchEventPublisher.java index dfc1a9b31d9..88370588240 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchEventPublisher.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchEventPublisher.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.reindex.DeleteByQueryRequest; @@ -129,6 +130,8 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher { default: LOG.warn("Ignoring Entity Type {}", entityType); } + } catch (DocumentMissingException ex) { + LOG.error("Missing Document", ex); } catch (ElasticsearchException e) { LOG.error("failed to update ES doc"); LOG.debug(e.getMessage()); @@ -202,7 +205,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher { UpdateRequest updateRequest = new UpdateRequest(ElasticSearchIndexType.TABLE_SEARCH_INDEX.indexName, event.getEntityId().toString()); TableESIndex tableESIndex = null; - if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { + if (event.getEntity() != null + && event.getEventType() != EventType.ENTITY_SOFT_DELETED + && event.getEventType() != EventType.ENTITY_DELETED) { Table table = (Table) event.getEntity(); tableESIndex = TableESIndex.builder(table, event.getEventType()).build(); } @@ -237,7 +242,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher { UpdateRequest updateRequest = new UpdateRequest(ElasticSearchIndexType.TOPIC_SEARCH_INDEX.indexName, event.getEntityId().toString()); TopicESIndex topicESIndex = null; - if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { + if (event.getEntity() != null + && event.getEventType() != EventType.ENTITY_SOFT_DELETED + && event.getEventType() != EventType.ENTITY_DELETED) { Topic topic; topic = (Topic) event.getEntity(); topicESIndex = TopicESIndex.builder(topic, event.getEventType()).build(); @@ -273,7 +280,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher { DashboardESIndex dashboardESIndex = null; UpdateRequest updateRequest = new UpdateRequest(ElasticSearchIndexType.DASHBOARD_SEARCH_INDEX.indexName, event.getEntityId().toString()); - if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { + if (event.getEntity() != null + && event.getEventType() != EventType.ENTITY_SOFT_DELETED + && event.getEventType() != EventType.ENTITY_DELETED) { Dashboard dashboard = (Dashboard) event.getEntity(); dashboardESIndex = DashboardESIndex.builder(dashboard, event.getEventType()).build(); } @@ -306,7 +315,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher { private void updatePipeline(ChangeEvent event) throws IOException { PipelineESIndex pipelineESIndex = null; - if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { + if (event.getEntity() != null + && event.getEventType() != EventType.ENTITY_SOFT_DELETED + && event.getEventType() != EventType.ENTITY_DELETED) { Pipeline pipeline = (Pipeline) event.getEntity(); pipelineESIndex = PipelineESIndex.builder(pipeline, event.getEventType()).build(); } @@ -343,7 +354,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher { UpdateRequest updateRequest = new UpdateRequest(ElasticSearchIndexType.USER_SEARCH_INDEX.indexName, event.getEntityId().toString()); UserESIndex userESIndex = null; - if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { + if (event.getEntity() != null + && event.getEventType() != EventType.ENTITY_SOFT_DELETED + && event.getEventType() != EventType.ENTITY_DELETED) { User user = (User) event.getEntity(); userESIndex = UserESIndex.builder(user).build(); } @@ -374,7 +387,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher { UpdateRequest updateRequest = new UpdateRequest(ElasticSearchIndexType.TEAM_SEARCH_INDEX.indexName, event.getEntityId().toString()); TeamESIndex teamESIndex = null; - if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { + if (event.getEntity() != null + && event.getEventType() != EventType.ENTITY_SOFT_DELETED + && event.getEventType() != EventType.ENTITY_DELETED) { Team team = (Team) event.getEntity(); teamESIndex = TeamESIndex.builder(team).build(); } @@ -405,7 +420,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher { UpdateRequest updateRequest = new UpdateRequest(ElasticSearchIndexType.GLOSSARY_SEARCH_INDEX.indexName, event.getEntityId().toString()); GlossaryTermESIndex glossaryESIndex = null; - if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { + if (event.getEntity() != null + && event.getEventType() != EventType.ENTITY_SOFT_DELETED + && event.getEventType() != EventType.ENTITY_DELETED) { GlossaryTerm glossaryTerm = (GlossaryTerm) event.getEntity(); glossaryESIndex = GlossaryTermESIndex.builder(glossaryTerm, event.getEventType()).build(); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java index 7ee2ec29eba..90e36bc717b 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java @@ -817,7 +817,7 @@ class GlossaryTermESIndex extends ElasticSearchIndex { String description = glossaryTerm.getDescription() != null ? glossaryTerm.getDescription() : ""; String displayName = glossaryTerm.getDisplayName() != null ? glossaryTerm.getDisplayName() : ""; return internalBuilder() - .glossaryId(glossaryTerm.getGlossary().getId().toString()) + .glossaryTermId(glossaryTerm.getId().toString()) .name(glossaryTerm.getName()) .displayName(displayName) .description(description) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/events/AbstractEventPublisher.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/events/AbstractEventPublisher.java index a1da0cc739f..7507e0c12d9 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/events/AbstractEventPublisher.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/events/AbstractEventPublisher.java @@ -59,7 +59,7 @@ public abstract class AbstractEventPublisher implements EventPublisher { } catch (Exception e) { LOG.error( "Failed to publish event type {} for entity {}", changeEvent.getEventType(), changeEvent.getEntityType()); - LOG.error(e.getMessage()); + LOG.error(e.getMessage(), e); } } diff --git a/ingestion/src/metadata/ingestion/source/sample_data.py b/ingestion/src/metadata/ingestion/source/sample_data.py index dd2d611b2c8..7f03a7220bc 100644 --- a/ingestion/src/metadata/ingestion/source/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/sample_data.py @@ -14,6 +14,7 @@ import json import logging import os import random +import sys import traceback import uuid from collections import namedtuple @@ -625,6 +626,8 @@ class SampleDataSource(Source[Entity]): yield OMetaUserProfile(user=user_metadata, teams=teams, roles=roles) except Exception as err: logger.error(err) + logger.debug(traceback.format_exc()) + logger.debug(sys.exc_info()[2]) def ingest_table_tests(self) -> Iterable[OMetaTableTest]: """