diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index f7b4db75a98..7a95918d214 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -42,7 +42,14 @@ public interface SearchClient String DEFAULT_UPDATE_SCRIPT = """ for (k in params.keySet()) { - ctx._source.put(k, params.get(k)) + if (k != 'fieldsToRemove') { + ctx._source.put(k, params.get(k)) + } + } + if (params.containsKey('fieldsToRemove')) { + for (field in params.fieldsToRemove) { + ctx._source.remove(field) + } } """; String REMOVE_DOMAINS_CHILDREN_SCRIPT = "ctx._source.remove('domain')"; @@ -387,6 +394,8 @@ public interface SearchClient "tier", "changeDescription"); + Set FIELDS_TO_REMOVE_WHEN_NULL = Set.of("tier", "certification"); + boolean isClientAvailable(); boolean isNewClientAvailable(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntityManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntityManager.java index d1551087c38..69c04173246 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntityManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntityManager.java @@ -4,6 +4,7 @@ import static org.openmetadata.service.exception.CatalogGenericExceptionMapper.g import static org.openmetadata.service.search.SearchClient.ADD_UPDATE_ENTITY_RELATIONSHIP; import static org.openmetadata.service.search.SearchClient.ADD_UPDATE_LINEAGE; import static org.openmetadata.service.search.SearchClient.DELETE_COLUMN_LINEAGE_SCRIPT; +import static org.openmetadata.service.search.SearchClient.FIELDS_TO_REMOVE_WHEN_NULL; import static org.openmetadata.service.search.SearchClient.UPDATE_COLUMN_LINEAGE_SCRIPT; import static org.openmetadata.service.search.SearchClient.UPDATE_FQN_PREFIX_SCRIPT; import static org.openmetadata.service.search.SearchClient.UPDATE_GLOSSARY_TERM_TAG_FQN_BY_PREFIX_SCRIPT; @@ -1034,9 +1035,27 @@ public class ElasticSearchEntityManager implements EntityManagementClient { } private Map convertToJsonDataMap(Map map) { - return JsonUtils.getMap(map).entrySet().stream() - .filter(entry -> entry.getValue() != null) - .collect(Collectors.toMap(Map.Entry::getKey, entry -> JsonData.of(entry.getValue()))); + List fieldsToRemove = new ArrayList<>(); + + Map result = + JsonUtils.getMap(map).entrySet().stream() + .filter( + entry -> { + if (entry.getValue() == null + && FIELDS_TO_REMOVE_WHEN_NULL.contains(entry.getKey())) { + fieldsToRemove.add(entry.getKey()); + return false; + } + return entry.getValue() != null; + }) + .collect(Collectors.toMap(Map.Entry::getKey, entry -> JsonData.of(entry.getValue()))); + + if (!fieldsToRemove.isEmpty()) { + result.put("fieldsToRemove", JsonData.of(fieldsToRemove)); + LOG.info("Fields to remove from search index: {}", fieldsToRemove); + } + + return result; } private JsonData toJsonData(String doc) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java index 1dac96a4460..eb2d3fe32ab 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java @@ -125,11 +125,8 @@ public interface SearchIndex { map.put("fqnParts", getFQNParts(entity.getFullyQualifiedName())); map.put("deleted", entity.getDeleted() != null && entity.getDeleted()); TagLabel tierTag = new ParseTags(Entity.getEntityTags(entityType, entity)).getTierTag(); - Optional.ofNullable(tierTag) - .filter(tier -> tier.getTagFQN() != null && !tier.getTagFQN().isEmpty()) - .ifPresent(tier -> map.put("tier", tier)); - Optional.ofNullable(entity.getCertification()) - .ifPresent(assetCertification -> map.put("certification", assetCertification)); + map.put("tier", tierTag); + map.put("certification", entity.getCertification()); return map; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntityManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntityManager.java index b19bf857b0a..e6248369957 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntityManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntityManager.java @@ -4,6 +4,7 @@ import static org.openmetadata.service.exception.CatalogGenericExceptionMapper.g import static org.openmetadata.service.search.SearchClient.ADD_UPDATE_ENTITY_RELATIONSHIP; import static org.openmetadata.service.search.SearchClient.ADD_UPDATE_LINEAGE; import static org.openmetadata.service.search.SearchClient.DELETE_COLUMN_LINEAGE_SCRIPT; +import static org.openmetadata.service.search.SearchClient.FIELDS_TO_REMOVE_WHEN_NULL; import static org.openmetadata.service.search.SearchClient.UPDATE_COLUMN_LINEAGE_SCRIPT; import static org.openmetadata.service.search.SearchClient.UPDATE_FQN_PREFIX_SCRIPT; import static org.openmetadata.service.search.SearchClient.UPDATE_GLOSSARY_TERM_TAG_FQN_BY_PREFIX_SCRIPT; @@ -1025,9 +1026,27 @@ public class OpenSearchEntityManager implements EntityManagementClient { } private Map convertToJsonDataMap(Map map) { - return JsonUtils.getMap(map).entrySet().stream() - .filter(entry -> entry.getValue() != null) - .collect(Collectors.toMap(Map.Entry::getKey, entry -> JsonData.of(entry.getValue()))); + List fieldsToRemove = new ArrayList<>(); + + Map result = + JsonUtils.getMap(map).entrySet().stream() + .filter( + entry -> { + if (entry.getValue() == null + && FIELDS_TO_REMOVE_WHEN_NULL.contains(entry.getKey())) { + fieldsToRemove.add(entry.getKey()); + return false; + } + return entry.getValue() != null; + }) + .collect(Collectors.toMap(Map.Entry::getKey, entry -> JsonData.of(entry.getValue()))); + + if (!fieldsToRemove.isEmpty()) { + result.put("fieldsToRemove", JsonData.of(fieldsToRemove)); + LOG.info("Fields to remove from search index: {}", fieldsToRemove); + } + + return result; } private JsonData toJsonData(String doc) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index 04f7e505702..f752b9dec0d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -94,7 +94,17 @@ import java.io.IOException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.text.ParseException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -2462,6 +2472,122 @@ public class TableResourceTest extends EntityResourceTest { CatalogExceptionMessage.mutuallyExclusiveLabels(TIER2_TAG_LABEL, TIER1_TAG_LABEL)); } + @Test + void test_tierRemovalFromDatabaseAndSearch(TestInfo test) + throws IOException, InterruptedException { + Table table = createEntity(createRequest(test), ADMIN_AUTH_HEADERS); + + String originalJson = JsonUtils.pojoToJson(table); + table.withTags(List.of(TIER1_TAG_LABEL)); + + ChangeDescription change = getChangeDescription(table, MINOR_UPDATE); + fieldAdded(change, FIELD_TAGS, List.of(TIER1_TAG_LABEL)); + table = patchEntityAndCheck(table, originalJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); + + assertNotNull(table.getTags()); + assertEquals(1, table.getTags().size()); + assertEquals(TIER1_TAG_LABEL.getTagFQN(), table.getTags().get(0).getTagFQN()); + + Thread.sleep(2000); + + assertTierInSearch(table, TIER1_TAG_LABEL.getTagFQN()); + + originalJson = JsonUtils.pojoToJson(table); + table.withTags(new ArrayList<>()); + + change = getChangeDescription(table, MINOR_UPDATE); + change.setPreviousVersion(table.getVersion()); + fieldDeleted(change, FIELD_TAGS, List.of(TIER1_TAG_LABEL)); + table = patchEntityAndCheck(table, originalJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); + + assertTrue(table.getTags() == null || table.getTags().isEmpty()); + + Thread.sleep(2000); + + assertTierNotInSearch(table); + } + + private void assertTierInSearch(Table table, String expectedTierFqn) throws IOException { + RestClient searchClient = getSearchClient(); + IndexMapping index = Entity.getSearchRepository().getIndexMapping(TABLE); + + Request refreshRequest = + new Request( + "POST", + format( + "%s/_refresh", index.getIndexName(Entity.getSearchRepository().getClusterAlias()))); + searchClient.performRequest(refreshRequest); + + Request request = + new Request( + "GET", + format( + "%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias()))); + String query = + format( + "{\"size\": 1, \"query\": {\"bool\": {\"must\": [{\"term\": {\"_id\": \"%s\"}}]}}}", + table.getId().toString()); + request.setJsonEntity(query); + + Response response = searchClient.performRequest(request); + String jsonString = EntityUtils.toString(response.getEntity()); + HashMap map = + (HashMap) JsonUtils.readOrConvertValue(jsonString, HashMap.class); + LinkedHashMap hits = (LinkedHashMap) map.get("hits"); + ArrayList> hitsList = + (ArrayList>) hits.get("hits"); + + assertFalse(hitsList.isEmpty(), "Table should be found in search index"); + + Map source = (Map) hitsList.get(0).get("_source"); + Map tier = (Map) source.get("tier"); + + assertNotNull(tier, "Tier should be present in search index"); + assertEquals(expectedTierFqn, tier.get("tagFQN"), "Tier tag FQN should match in search index"); + + searchClient.close(); + } + + private void assertTierNotInSearch(Table table) throws IOException { + RestClient searchClient = getSearchClient(); + IndexMapping index = Entity.getSearchRepository().getIndexMapping(TABLE); + + Request refreshRequest = + new Request( + "POST", + format( + "%s/_refresh", index.getIndexName(Entity.getSearchRepository().getClusterAlias()))); + searchClient.performRequest(refreshRequest); + + Request request = + new Request( + "GET", + format( + "%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias()))); + String query = + format( + "{\"size\": 1, \"query\": {\"bool\": {\"must\": [{\"term\": {\"_id\": \"%s\"}}]}}}", + table.getId().toString()); + request.setJsonEntity(query); + + Response response = searchClient.performRequest(request); + String jsonString = EntityUtils.toString(response.getEntity()); + HashMap map = + (HashMap) JsonUtils.readOrConvertValue(jsonString, HashMap.class); + LinkedHashMap hits = (LinkedHashMap) map.get("hits"); + ArrayList> hitsList = + (ArrayList>) hits.get("hits"); + + assertFalse(hitsList.isEmpty(), "Table should be found in search index"); + + Map source = (Map) hitsList.get(0).get("_source"); + Object tier = source.get("tier"); + + assertNull(tier, "Tier should be removed from search index"); + + searchClient.close(); + } + @Test void test_ownershipInheritance(TestInfo test) throws IOException { // When a databaseSchema has no owner set, it inherits the ownership from database