yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-07-30 12:49:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #4446: Fix Glossary Deletion to update ES Index (#4447)

Browse Source
This commit is contained in:
Sriharsha Chintalapani 2022-04-24 12:09:12 -07:00 committed by GitHub
parent 336e97a643
commit feafa448e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 29 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchEventPublisher.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.engine.DocumentMissingException;
import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.index.reindex.DeleteByQueryRequest;
@ -129,6 +130,8 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher {
default: default:
LOG.warn("Ignoring Entity Type {}", entityType); LOG.warn("Ignoring Entity Type {}", entityType);
} }
} catch (DocumentMissingException ex) {
LOG.error("Missing Document", ex);
} catch (ElasticsearchException e) { } catch (ElasticsearchException e) {
LOG.error("failed to update ES doc"); LOG.error("failed to update ES doc");
LOG.debug(e.getMessage()); LOG.debug(e.getMessage());
@ -202,7 +205,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher {
UpdateRequest updateRequest = UpdateRequest updateRequest =
new UpdateRequest(ElasticSearchIndexType.TABLE_SEARCH_INDEX.indexName, event.getEntityId().toString()); new UpdateRequest(ElasticSearchIndexType.TABLE_SEARCH_INDEX.indexName, event.getEntityId().toString());
TableESIndex tableESIndex = null; TableESIndex tableESIndex = null;
if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { if (event.getEntity() != null
&& event.getEventType() != EventType.ENTITY_SOFT_DELETED
&& event.getEventType() != EventType.ENTITY_DELETED) {
Table table = (Table) event.getEntity(); Table table = (Table) event.getEntity();
tableESIndex = TableESIndex.builder(table, event.getEventType()).build(); tableESIndex = TableESIndex.builder(table, event.getEventType()).build();
} }
@ -237,7 +242,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher {
UpdateRequest updateRequest = UpdateRequest updateRequest =
new UpdateRequest(ElasticSearchIndexType.TOPIC_SEARCH_INDEX.indexName, event.getEntityId().toString()); new UpdateRequest(ElasticSearchIndexType.TOPIC_SEARCH_INDEX.indexName, event.getEntityId().toString());
TopicESIndex topicESIndex = null; TopicESIndex topicESIndex = null;
if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { if (event.getEntity() != null
&& event.getEventType() != EventType.ENTITY_SOFT_DELETED
&& event.getEventType() != EventType.ENTITY_DELETED) {
Topic topic; Topic topic;
topic = (Topic) event.getEntity(); topic = (Topic) event.getEntity();
topicESIndex = TopicESIndex.builder(topic, event.getEventType()).build(); topicESIndex = TopicESIndex.builder(topic, event.getEventType()).build();
@ -273,7 +280,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher {
DashboardESIndex dashboardESIndex = null; DashboardESIndex dashboardESIndex = null;
UpdateRequest updateRequest = UpdateRequest updateRequest =
new UpdateRequest(ElasticSearchIndexType.DASHBOARD_SEARCH_INDEX.indexName, event.getEntityId().toString()); new UpdateRequest(ElasticSearchIndexType.DASHBOARD_SEARCH_INDEX.indexName, event.getEntityId().toString());
if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { if (event.getEntity() != null
&& event.getEventType() != EventType.ENTITY_SOFT_DELETED
&& event.getEventType() != EventType.ENTITY_DELETED) {
Dashboard dashboard = (Dashboard) event.getEntity(); Dashboard dashboard = (Dashboard) event.getEntity();
dashboardESIndex = DashboardESIndex.builder(dashboard, event.getEventType()).build(); dashboardESIndex = DashboardESIndex.builder(dashboard, event.getEventType()).build();
} }
@ -306,7 +315,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher {
private void updatePipeline(ChangeEvent event) throws IOException { private void updatePipeline(ChangeEvent event) throws IOException {
PipelineESIndex pipelineESIndex = null; PipelineESIndex pipelineESIndex = null;
if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { if (event.getEntity() != null
&& event.getEventType() != EventType.ENTITY_SOFT_DELETED
&& event.getEventType() != EventType.ENTITY_DELETED) {
Pipeline pipeline = (Pipeline) event.getEntity(); Pipeline pipeline = (Pipeline) event.getEntity();
pipelineESIndex = PipelineESIndex.builder(pipeline, event.getEventType()).build(); pipelineESIndex = PipelineESIndex.builder(pipeline, event.getEventType()).build();
} }
@ -343,7 +354,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher {
UpdateRequest updateRequest = UpdateRequest updateRequest =
new UpdateRequest(ElasticSearchIndexType.USER_SEARCH_INDEX.indexName, event.getEntityId().toString()); new UpdateRequest(ElasticSearchIndexType.USER_SEARCH_INDEX.indexName, event.getEntityId().toString());
UserESIndex userESIndex = null; UserESIndex userESIndex = null;
if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { if (event.getEntity() != null
&& event.getEventType() != EventType.ENTITY_SOFT_DELETED
&& event.getEventType() != EventType.ENTITY_DELETED) {
User user = (User) event.getEntity(); User user = (User) event.getEntity();
userESIndex = UserESIndex.builder(user).build(); userESIndex = UserESIndex.builder(user).build();
} }
@ -374,7 +387,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher {
UpdateRequest updateRequest = UpdateRequest updateRequest =
new UpdateRequest(ElasticSearchIndexType.TEAM_SEARCH_INDEX.indexName, event.getEntityId().toString()); new UpdateRequest(ElasticSearchIndexType.TEAM_SEARCH_INDEX.indexName, event.getEntityId().toString());
TeamESIndex teamESIndex = null; TeamESIndex teamESIndex = null;
if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { if (event.getEntity() != null
&& event.getEventType() != EventType.ENTITY_SOFT_DELETED
&& event.getEventType() != EventType.ENTITY_DELETED) {
Team team = (Team) event.getEntity(); Team team = (Team) event.getEntity();
teamESIndex = TeamESIndex.builder(team).build(); teamESIndex = TeamESIndex.builder(team).build();
} }
@ -405,7 +420,9 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher {
UpdateRequest updateRequest = UpdateRequest updateRequest =
new UpdateRequest(ElasticSearchIndexType.GLOSSARY_SEARCH_INDEX.indexName, event.getEntityId().toString()); new UpdateRequest(ElasticSearchIndexType.GLOSSARY_SEARCH_INDEX.indexName, event.getEntityId().toString());
GlossaryTermESIndex glossaryESIndex = null; GlossaryTermESIndex glossaryESIndex = null;
if (event.getEntity() != null && event.getEventType() != EventType.ENTITY_SOFT_DELETED) { if (event.getEntity() != null
&& event.getEventType() != EventType.ENTITY_SOFT_DELETED
&& event.getEventType() != EventType.ENTITY_DELETED) {
GlossaryTerm glossaryTerm = (GlossaryTerm) event.getEntity(); GlossaryTerm glossaryTerm = (GlossaryTerm) event.getEntity();
glossaryESIndex = GlossaryTermESIndex.builder(glossaryTerm, event.getEventType()).build(); glossaryESIndex = GlossaryTermESIndex.builder(glossaryTerm, event.getEventType()).build();
} }

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java
View File

@ -817,7 +817,7 @@ class GlossaryTermESIndex extends ElasticSearchIndex {
String description = glossaryTerm.getDescription() != null ? glossaryTerm.getDescription() : ""; String description = glossaryTerm.getDescription() != null ? glossaryTerm.getDescription() : "";
String displayName = glossaryTerm.getDisplayName() != null ? glossaryTerm.getDisplayName() : ""; String displayName = glossaryTerm.getDisplayName() != null ? glossaryTerm.getDisplayName() : "";
return internalBuilder() return internalBuilder()
.glossaryId(glossaryTerm.getGlossary().getId().toString()) .glossaryTermId(glossaryTerm.getId().toString())
.name(glossaryTerm.getName()) .name(glossaryTerm.getName())
.displayName(displayName) .displayName(displayName)
.description(description) .description(description)

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/events/AbstractEventPublisher.java
View File

@ -59,7 +59,7 @@ public abstract class AbstractEventPublisher implements EventPublisher {
} catch (Exception e) { } catch (Exception e) {
LOG.error( LOG.error(
"Failed to publish event type {} for entity {}", changeEvent.getEventType(), changeEvent.getEntityType()); "Failed to publish event type {} for entity {}", changeEvent.getEventType(), changeEvent.getEntityType());
LOG.error(e.getMessage()); LOG.error(e.getMessage(), e);
} }
} }

3
ingestion/src/metadata/ingestion/source/sample_data.py
View File

@ -14,6 +14,7 @@ import json
import logging import logging
import os import os
import random import random
import sys
import traceback import traceback
import uuid import uuid
from collections import namedtuple from collections import namedtuple
@ -625,6 +626,8 @@ class SampleDataSource(Source[Entity]):
yield OMetaUserProfile(user=user_metadata, teams=teams, roles=roles) yield OMetaUserProfile(user=user_metadata, teams=teams, roles=roles)
except Exception as err: except Exception as err:
logger.error(err) logger.error(err)
logger.debug(traceback.format_exc())
logger.debug(sys.exc_info()[2])
def ingest_table_tests(self) -> Iterable[OMetaTableTest]: def ingest_table_tests(self) -> Iterable[OMetaTableTest]:
""" """