From fcb06383c1b298742c46a2d98674494830bb69b7 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Thu, 12 Oct 2023 16:11:24 -0700 Subject: [PATCH] Disable tags when the classification is disabled (#13532) * Disable tags when the classification is disabled * remove unnecessary constants --------- Co-authored-by: 07Himank <112613760+07Himank@users.noreply.github.com> --- .../java/org/openmetadata/service/Entity.java | 2 + .../service/search/SearchClient.java | 15 ++-- .../service/search/SearchRepository.java | 72 ++++++++++++------- .../elasticsearch/ElasticSearchClient.java | 5 +- .../search/opensearch/OpenSearchClient.java | 5 +- .../resources/elasticsearch/indexMapping.json | 2 +- 6 files changed, 62 insertions(+), 39 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 20c99d658ee..d6a01e08e9e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -108,6 +108,8 @@ public final class Entity { public static final String FIELD_LIFE_CYCLE = "lifeCycle"; + public static final String FIELD_DISABLED = "disabled"; + // // Service entities // diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index bcc88971e25..54f85783cbc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -13,7 +13,6 @@ import javax.ws.rs.core.Response; import org.apache.commons.lang3.tuple.Pair; import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; -import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.exception.CustomExceptionMessage; import org.openmetadata.service.search.models.IndexMapping; import org.openmetadata.service.util.SSLUtil; @@ -29,14 +28,16 @@ public interface SearchClient { String DELETE = "delete"; String GLOBAL_SEARCH_ALIAS = "AllEntities"; - String DEFAULT_UPDATE_SCRIPT = "for (k in params.keySet()) { ctx._source.put(k, params.get(k)) }"; - String CLASSIFICATION_DISABLE_SCRIPT = "ctx._source.disabled=%s"; String REMOVE_DOMAINS_CHILDREN_SCRIPT = "ctx._source.remove('domain')"; - String PROPAGATE_FIELD_SCRIPT = "if(ctx._source.%s == null){ ctx._source.put('%s', params)}"; - String REMOVE_PROPAGATED_FIELD_SCRIPT = + String PROPAGATE_ENTITY_REFERENCE_FIELD_SCRIPT = "if(ctx._source.%s == null){ ctx._source.put('%s', params)}"; + + String PROPAGATE_FIELD_SCRIPT = "ctx._source.put('%s', '%s')"; + + String REMOVE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT = "if((ctx._source.%s != null) && (ctx._source.%s.id == '%s')){ ctx._source.remove('%s')}"; - String UPDATE_PROPAGATED_FIELD_SCRIPT = + String REMOVE_PROPAGATED_FIELD_SCRIPT = "ctx._source.remove('%s')"; + String UPDATE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT = "if((ctx._source.%s == null) || (ctx._source.%s.id == '%s')) { ctx._source.put('%s', params)}"; String SOFT_DELETE_RESTORE_SCRIPT = "ctx._source.put('deleted', '%s')"; String REMOVE_TAGS_CHILDREN_SCRIPT = @@ -84,7 +85,7 @@ public interface SearchClient { void softDeleteOrRestoreChildren(String indexName, String scriptTxt, List> fieldAndValue); - void updateChildren(String indexName, Pair fieldAndValue, Pair updates); + void updateChildren(String indexName, Pair fieldAndValue, Pair> updates); TreeMap> getSortedDate( String team, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 2bffd31654a..26fa1d086e3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -5,13 +5,15 @@ import static org.openmetadata.service.Entity.FIELD_USAGE_SUMMARY; import static org.openmetadata.service.Entity.QUERY; import static org.openmetadata.service.search.SearchClient.DEFAULT_UPDATE_SCRIPT; import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS; +import static org.openmetadata.service.search.SearchClient.PROPAGATE_ENTITY_REFERENCE_FIELD_SCRIPT; import static org.openmetadata.service.search.SearchClient.PROPAGATE_FIELD_SCRIPT; import static org.openmetadata.service.search.SearchClient.REMOVE_DOMAINS_CHILDREN_SCRIPT; +import static org.openmetadata.service.search.SearchClient.REMOVE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT; import static org.openmetadata.service.search.SearchClient.REMOVE_PROPAGATED_FIELD_SCRIPT; import static org.openmetadata.service.search.SearchClient.REMOVE_TAGS_CHILDREN_SCRIPT; import static org.openmetadata.service.search.SearchClient.REMOVE_TEST_SUITE_CHILDREN_SCRIPT; import static org.openmetadata.service.search.SearchClient.SOFT_DELETE_RESTORE_SCRIPT; -import static org.openmetadata.service.search.SearchClient.UPDATE_PROPAGATED_FIELD_SCRIPT; +import static org.openmetadata.service.search.SearchClient.UPDATE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT; import com.fasterxml.jackson.core.type.TypeReference; import java.io.IOException; @@ -43,6 +45,7 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.FieldChange; import org.openmetadata.schema.type.UsageDetails; import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.UnhandledServerException; import org.openmetadata.service.search.elasticsearch.ElasticSearchClient; import org.openmetadata.service.search.indexes.SearchIndex; import org.openmetadata.service.search.models.IndexMapping; @@ -58,7 +61,8 @@ public class SearchRepository { private final String language; - private final List inheritableFields = List.of(Entity.FIELD_OWNER, Entity.FIELD_DOMAIN); + private final List inheritableFields = + List.of(Entity.FIELD_OWNER, Entity.FIELD_DOMAIN, Entity.FIELD_DISABLED); public final List dataInsightReports = List.of( @@ -252,7 +256,7 @@ public class SearchRepository { public void propagateInheritedFieldsToChildren( String entityType, String entityId, ChangeDescription changeDescription, IndexMapping indexMapping) { if (changeDescription != null) { - Pair updates = getInheritedFieldChanges(changeDescription); + Pair> updates = getInheritedFieldChanges(changeDescription); Pair parentMatch = new ImmutablePair<>(entityType + ".id", entityId); if (updates.getKey() != null && !updates.getKey().isEmpty()) { searchClient.updateChildren(indexMapping.getAlias(), parentMatch, updates); @@ -260,40 +264,58 @@ public class SearchRepository { } } - private Pair getInheritedFieldChanges(ChangeDescription changeDescription) { + private Pair> getInheritedFieldChanges(ChangeDescription changeDescription) { StringBuilder scriptTxt = new StringBuilder(); - EntityReference fieldData = null; + Map fieldData = new HashMap<>(); if (changeDescription != null) { for (FieldChange field : changeDescription.getFieldsAdded()) { if (inheritableFields.contains(field.getName())) { - scriptTxt.append(String.format(PROPAGATE_FIELD_SCRIPT, field.getName(), field.getName())); - fieldData = JsonUtils.readValue(field.getNewValue().toString(), EntityReference.class); + try { + EntityReference entityReference = + JsonUtils.readValue(field.getNewValue().toString(), EntityReference.class); + scriptTxt.append(String.format(PROPAGATE_ENTITY_REFERENCE_FIELD_SCRIPT, field.getName(), field.getName())); + fieldData = JsonUtils.getMap(entityReference); + } catch (UnhandledServerException e) { + scriptTxt.append(String.format(PROPAGATE_FIELD_SCRIPT, field.getName(), field.getNewValue())); + } } } for (FieldChange field : changeDescription.getFieldsUpdated()) { if (inheritableFields.contains(field.getName())) { - EntityReference entityReference = JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class); - scriptTxt.append( - String.format( - UPDATE_PROPAGATED_FIELD_SCRIPT, - field.getName(), - field.getName(), - entityReference.getId().toString(), - field.getName())); - fieldData = JsonUtils.readValue(field.getNewValue().toString(), EntityReference.class); + try { + EntityReference oldEntityReference = + JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class); + EntityReference newEntityReference = + JsonUtils.readValue(field.getNewValue().toString(), EntityReference.class); + scriptTxt.append( + String.format( + UPDATE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT, + field.getName(), + field.getName(), + oldEntityReference.getId().toString(), + field.getName())); + fieldData = JsonUtils.getMap(newEntityReference); + } catch (UnhandledServerException e) { + scriptTxt.append(String.format(PROPAGATE_FIELD_SCRIPT, field.getName(), field.getNewValue())); + } } } for (FieldChange field : changeDescription.getFieldsDeleted()) { if (inheritableFields.contains(field.getName())) { - EntityReference entityReference = JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class); - scriptTxt.append( - String.format( - REMOVE_PROPAGATED_FIELD_SCRIPT, - field.getName(), - field.getName(), - entityReference.getId().toString(), - field.getName())); - fieldData = (EntityReference) field.getNewValue(); + try { + EntityReference entityReference = + JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class); + scriptTxt.append( + String.format( + REMOVE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT, + field.getName(), + field.getName(), + entityReference.getId().toString(), + field.getName())); + fieldData = JsonUtils.getMap(entityReference); + } catch (UnhandledServerException e) { + scriptTxt.append(String.format(REMOVE_PROPAGATED_FIELD_SCRIPT, field.getName())); + } } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 3dd227a8a4a..355cf618201 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -111,7 +111,6 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.openmetadata.schema.DataInsightInterface; import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; -import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface; import org.openmetadata.service.jdbi3.DataInsightChartRepository; import org.openmetadata.service.search.SearchClient; @@ -981,7 +980,7 @@ public class ElasticSearchClient implements SearchClient { @Override public void updateChildren( - String indexName, Pair fieldAndValue, Pair updates) { + String indexName, Pair fieldAndValue, Pair> updates) { if (isClientAvailable) { UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); updateByQueryRequest.setQuery( @@ -991,7 +990,7 @@ public class ElasticSearchClient implements SearchClient { ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, updates.getKey(), - JsonUtils.getMap(updates.getValue() == null ? new HashMap<>() : JsonUtils.getMap(updates.getValue()))); + JsonUtils.getMap(updates.getValue() == null ? new HashMap<>() : updates.getValue())); updateByQueryRequest.setScript(script); updateElasticSearchByQuery(updateByQueryRequest); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 800445bf252..21565b0ad6e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -42,7 +42,6 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.openmetadata.schema.DataInsightInterface; import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; -import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface; import org.openmetadata.service.jdbi3.DataInsightChartRepository; import org.openmetadata.service.search.SearchClient; @@ -988,7 +987,7 @@ public class OpenSearchClient implements SearchClient { @Override public void updateChildren( - String indexName, Pair fieldAndValue, Pair updates) { + String indexName, Pair fieldAndValue, Pair> updates) { if (isClientAvailable) { UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); updateByQueryRequest.setQuery(new MatchQueryBuilder(fieldAndValue.getKey(), fieldAndValue.getValue())); @@ -997,7 +996,7 @@ public class OpenSearchClient implements SearchClient { ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, updates.getKey(), - JsonUtils.getMap(updates.getValue() == null ? new HashMap<>() : updates.getValue())); + updates.getValue() == null ? new HashMap<>() : updates.getValue()); updateByQueryRequest.setScript(script); updateOpenSearchByQuery(updateByQueryRequest); } diff --git a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json index 7af6fc602b5..87f826c518c 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json @@ -111,7 +111,7 @@ "indexName": "tag_search_index", "indexMappingFile": "/elasticsearch/%s/tag_index_mapping.json", "alias": "tag", - "parentAliases": ["all"] + "parentAliases": ["classification","all"] }, "classification": { "indexName": "classification_search_index",