yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-13 00:22:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix issue for delete tier and certification not removed from search data (#24441)

Browse Source 
* Fix issue for delete tier and certification not removed from search data

* Added test for tier remove case
This commit is contained in:
Bhanu Agrawal 2025-11-20 11:45:48 +05:30 committed by GitHub
parent 6ec36b1b81
commit b1f885a573
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 183 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
View File

@ -42,7 +42,14 @@ public interface SearchClient
String DEFAULT_UPDATE_SCRIPT =
"""
for (k in params.keySet()) {
ctx._source.put(k, params.get(k))
if (k != 'fieldsToRemove') {
ctx._source.put(k, params.get(k))
}
}
if (params.containsKey('fieldsToRemove')) {
for (field in params.fieldsToRemove) {
ctx._source.remove(field)
}
}
""";
String REMOVE_DOMAINS_CHILDREN_SCRIPT = "ctx._source.remove('domain')";
@ -387,6 +394,8 @@ public interface SearchClient
"tier",
"changeDescription");
Set<String> FIELDS_TO_REMOVE_WHEN_NULL = Set.of("tier", "certification");
boolean isClientAvailable();
boolean isNewClientAvailable();

25
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntityManager.java
View File

@ -4,6 +4,7 @@ import static org.openmetadata.service.exception.CatalogGenericExceptionMapper.g
import static org.openmetadata.service.search.SearchClient.ADD_UPDATE_ENTITY_RELATIONSHIP;
import static org.openmetadata.service.search.SearchClient.ADD_UPDATE_LINEAGE;
import static org.openmetadata.service.search.SearchClient.DELETE_COLUMN_LINEAGE_SCRIPT;
import static org.openmetadata.service.search.SearchClient.FIELDS_TO_REMOVE_WHEN_NULL;
import static org.openmetadata.service.search.SearchClient.UPDATE_COLUMN_LINEAGE_SCRIPT;
import static org.openmetadata.service.search.SearchClient.UPDATE_FQN_PREFIX_SCRIPT;
import static org.openmetadata.service.search.SearchClient.UPDATE_GLOSSARY_TERM_TAG_FQN_BY_PREFIX_SCRIPT;
@ -1034,9 +1035,27 @@ public class ElasticSearchEntityManager implements EntityManagementClient {
}
private Map<String, JsonData> convertToJsonDataMap(Map<String, Object> map) {
return JsonUtils.getMap(map).entrySet().stream()
.filter(entry -> entry.getValue() != null)
.collect(Collectors.toMap(Map.Entry::getKey, entry -> JsonData.of(entry.getValue())));
List<String> fieldsToRemove = new ArrayList<>();
Map<String, JsonData> result =
JsonUtils.getMap(map).entrySet().stream()
.filter(
entry -> {
if (entry.getValue() == null
&& FIELDS_TO_REMOVE_WHEN_NULL.contains(entry.getKey())) {
fieldsToRemove.add(entry.getKey());
return false;
}
return entry.getValue() != null;
})
.collect(Collectors.toMap(Map.Entry::getKey, entry -> JsonData.of(entry.getValue())));
if (!fieldsToRemove.isEmpty()) {
result.put("fieldsToRemove", JsonData.of(fieldsToRemove));
LOG.info("Fields to remove from search index: {}", fieldsToRemove);
}
return result;
}
private JsonData toJsonData(String doc) {

7
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java
View File

@ -125,11 +125,8 @@ public interface SearchIndex {
map.put("fqnParts", getFQNParts(entity.getFullyQualifiedName()));
map.put("deleted", entity.getDeleted() != null && entity.getDeleted());
TagLabel tierTag = new ParseTags(Entity.getEntityTags(entityType, entity)).getTierTag();
Optional.ofNullable(tierTag)
.filter(tier -> tier.getTagFQN() != null && !tier.getTagFQN().isEmpty())
.ifPresent(tier -> map.put("tier", tier));
Optional.ofNullable(entity.getCertification())
.ifPresent(assetCertification -> map.put("certification", assetCertification));
map.put("tier", tierTag);
map.put("certification", entity.getCertification());
return map;
}

25
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntityManager.java
View File

@ -4,6 +4,7 @@ import static org.openmetadata.service.exception.CatalogGenericExceptionMapper.g
import static org.openmetadata.service.search.SearchClient.ADD_UPDATE_ENTITY_RELATIONSHIP;
import static org.openmetadata.service.search.SearchClient.ADD_UPDATE_LINEAGE;
import static org.openmetadata.service.search.SearchClient.DELETE_COLUMN_LINEAGE_SCRIPT;
import static org.openmetadata.service.search.SearchClient.FIELDS_TO_REMOVE_WHEN_NULL;
import static org.openmetadata.service.search.SearchClient.UPDATE_COLUMN_LINEAGE_SCRIPT;
import static org.openmetadata.service.search.SearchClient.UPDATE_FQN_PREFIX_SCRIPT;
import static org.openmetadata.service.search.SearchClient.UPDATE_GLOSSARY_TERM_TAG_FQN_BY_PREFIX_SCRIPT;
@ -1025,9 +1026,27 @@ public class OpenSearchEntityManager implements EntityManagementClient {
}
private Map<String, JsonData> convertToJsonDataMap(Map<String, Object> map) {
return JsonUtils.getMap(map).entrySet().stream()
.filter(entry -> entry.getValue() != null)
.collect(Collectors.toMap(Map.Entry::getKey, entry -> JsonData.of(entry.getValue())));
List<String> fieldsToRemove = new ArrayList<>();
Map<String, JsonData> result =
JsonUtils.getMap(map).entrySet().stream()
.filter(
entry -> {
if (entry.getValue() == null
&& FIELDS_TO_REMOVE_WHEN_NULL.contains(entry.getKey())) {
fieldsToRemove.add(entry.getKey());
return false;
}
return entry.getValue() != null;
})
.collect(Collectors.toMap(Map.Entry::getKey, entry -> JsonData.of(entry.getValue())));
if (!fieldsToRemove.isEmpty()) {
result.put("fieldsToRemove", JsonData.of(fieldsToRemove));
LOG.info("Fields to remove from search index: {}", fieldsToRemove);
}
return result;
}
private JsonData toJsonData(String doc) {

128
openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java
View File

@ -94,7 +94,17 @@ import java.io.IOException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@ -2462,6 +2472,122 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
CatalogExceptionMessage.mutuallyExclusiveLabels(TIER2_TAG_LABEL, TIER1_TAG_LABEL));
}
@Test
void test_tierRemovalFromDatabaseAndSearch(TestInfo test)
throws IOException, InterruptedException {
Table table = createEntity(createRequest(test), ADMIN_AUTH_HEADERS);
String originalJson = JsonUtils.pojoToJson(table);
table.withTags(List.of(TIER1_TAG_LABEL));
ChangeDescription change = getChangeDescription(table, MINOR_UPDATE);
fieldAdded(change, FIELD_TAGS, List.of(TIER1_TAG_LABEL));
table = patchEntityAndCheck(table, originalJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
assertNotNull(table.getTags());
assertEquals(1, table.getTags().size());
assertEquals(TIER1_TAG_LABEL.getTagFQN(), table.getTags().get(0).getTagFQN());
Thread.sleep(2000);
assertTierInSearch(table, TIER1_TAG_LABEL.getTagFQN());
originalJson = JsonUtils.pojoToJson(table);
table.withTags(new ArrayList<>());
change = getChangeDescription(table, MINOR_UPDATE);
change.setPreviousVersion(table.getVersion());
fieldDeleted(change, FIELD_TAGS, List.of(TIER1_TAG_LABEL));
table = patchEntityAndCheck(table, originalJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
assertTrue(table.getTags() == null || table.getTags().isEmpty());
Thread.sleep(2000);
assertTierNotInSearch(table);
}
private void assertTierInSearch(Table table, String expectedTierFqn) throws IOException {
RestClient searchClient = getSearchClient();
IndexMapping index = Entity.getSearchRepository().getIndexMapping(TABLE);
Request refreshRequest =
new Request(
"POST",
format(
"%s/_refresh", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
searchClient.performRequest(refreshRequest);
Request request =
new Request(
"GET",
format(
"%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
String query =
format(
"{\"size\": 1, \"query\": {\"bool\": {\"must\": [{\"term\": {\"_id\": \"%s\"}}]}}}",
table.getId().toString());
request.setJsonEntity(query);
Response response = searchClient.performRequest(request);
String jsonString = EntityUtils.toString(response.getEntity());
HashMap<String, Object> map =
(HashMap<String, Object>) JsonUtils.readOrConvertValue(jsonString, HashMap.class);
LinkedHashMap<String, Object> hits = (LinkedHashMap<String, Object>) map.get("hits");
ArrayList<LinkedHashMap<String, Object>> hitsList =
(ArrayList<LinkedHashMap<String, Object>>) hits.get("hits");
assertFalse(hitsList.isEmpty(), "Table should be found in search index");
Map<String, Object> source = (Map<String, Object>) hitsList.get(0).get("_source");
Map<String, Object> tier = (Map<String, Object>) source.get("tier");
assertNotNull(tier, "Tier should be present in search index");
assertEquals(expectedTierFqn, tier.get("tagFQN"), "Tier tag FQN should match in search index");
searchClient.close();
}
private void assertTierNotInSearch(Table table) throws IOException {
RestClient searchClient = getSearchClient();
IndexMapping index = Entity.getSearchRepository().getIndexMapping(TABLE);
Request refreshRequest =
new Request(
"POST",
format(
"%s/_refresh", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
searchClient.performRequest(refreshRequest);
Request request =
new Request(
"GET",
format(
"%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
String query =
format(
"{\"size\": 1, \"query\": {\"bool\": {\"must\": [{\"term\": {\"_id\": \"%s\"}}]}}}",
table.getId().toString());
request.setJsonEntity(query);
Response response = searchClient.performRequest(request);
String jsonString = EntityUtils.toString(response.getEntity());
HashMap<String, Object> map =
(HashMap<String, Object>) JsonUtils.readOrConvertValue(jsonString, HashMap.class);
LinkedHashMap<String, Object> hits = (LinkedHashMap<String, Object>) map.get("hits");
ArrayList<LinkedHashMap<String, Object>> hitsList =
(ArrayList<LinkedHashMap<String, Object>>) hits.get("hits");
assertFalse(hitsList.isEmpty(), "Table should be found in search index");
Map<String, Object> source = (Map<String, Object>) hitsList.get(0).get("_source");
Object tier = source.get("tier");
assertNull(tier, "Tier should be removed from search index");
searchClient.close();
}
@Test
void test_ownershipInheritance(TestInfo test) throws IOException {
// When a databaseSchema has no owner set, it inherits the ownership from database