yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-07-30 20:59:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #15809 : Update asset indexes on glossary term hierarchy change/ term rename (#16083)

Browse Source 
* Fix #15809 : Update asset indexes on glossary term hierarchy change

* move logic to new method updateAssetIndexesOnGlossaryTermUpdate()

* enbale skip cypress test for the asset not found

* fix loss of tags and other relations while glossary parent change

---------

Co-authored-by: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com>
Co-authored-by: Ashish Gupta <ashish@getcollate.io>
This commit is contained in:
sonika-shah 2024-05-10 17:31:42 +05:30 committed by GitHub
parent 0ea05f8f8b
commit 3db9dadaa6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 197 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -2536,6 +2536,18 @@ public interface CollectionDAO {
updateTagPrefixInternal(update);
}
default void updateTargetFQNHashPrefix(
int source, String oldTargetFQNHashPrefix, String newTargetFQNHashPrefix) {
String update =
String.format(
"UPDATE tag_usage SET targetFQNHash = REPLACE(targetFQNHash, '%s.', '%s.') WHERE source = %s AND targetFQNHash LIKE '%s.%%'",
FullyQualifiedName.buildHash(oldTargetFQNHashPrefix),
FullyQualifiedName.buildHash(newTargetFQNHashPrefix),
source,
FullyQualifiedName.buildHash(oldTargetFQNHashPrefix));
updateTagPrefixInternal(update);
}
default void rename(int source, String oldFQN, String newFQN) {
renameInternal(source, oldFQN, newFQN, newFQN); // First rename tagFQN from oldFQN to newFQN
updateTagPrefix(
@ -2543,6 +2555,18 @@ public interface CollectionDAO {
newFQN); // Rename all the tagFQN prefixes starting with the oldFQN to newFQN
}
default void renameByTargetFQNHash(
int source, String oldTargetFQNHash, String newTargetFQNHash) {
renameByTargetFQNHashInternal(
source,
(oldTargetFQNHash),
newTargetFQNHash); // First rename targetFQN from oldFQN to newFQN
updateTargetFQNHashPrefix(
source,
oldTargetFQNHash,
newTargetFQNHash); // Rename all the targetFQN prefixes starting with the oldFQN to newFQN
}
/** Rename the tagFQN */
@SqlUpdate(
"Update tag_usage set tagFQN = :newFQN, tagFQNHash = :newFQNHash WHERE source = :source AND tagFQNHash = :oldFQNHash")
@ -2552,6 +2576,14 @@ public interface CollectionDAO {
@Bind("newFQN") String newFQN,
@BindFQN("newFQNHash") String newFQNHash);
/** Rename the targetFQN */
@SqlUpdate(
"Update tag_usage set targetFQNHash = :newTargetFQNHash WHERE source = :source AND targetFQNHash = :oldTargetFQNHash")
void renameByTargetFQNHashInternal(
@Bind("source") int source,
@BindFQN("oldTargetFQNHash") String oldTargetFQNHash,
@BindFQN("newTargetFQNHash") String newTargetFQNHash);
@SqlUpdate("<update>")
void updateTagPrefixInternal(@Define("update") String update);

101
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java
View File

@ -463,6 +463,59 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
return new HashMap<>();
}
private Map<String, EntityReference> getGlossaryTermsContainingFQNFromES(
String termFQN, int size) {
try {
String queryJson = String.format("fullyQualifiedName:%s*", termFQN);
SearchRequest searchRequest =
new SearchRequest.ElasticSearchRequestBuilder(
queryJson, size, "glossary_term_search_index")
.from(0)
.fetchSource(true)
.trackTotalHits(false)
.sortFieldParam("_score")
.deleted(false)
.sortOrder("desc")
.includeSourceFields(new ArrayList<>())
.build();
// Execute the search and parse the response
Response response = searchRepository.search(searchRequest);
String json = (String) response.getEntity();
Set<EntityReference> fqns = new TreeSet<>(compareEntityReferenceById);
// Extract hits from the response JSON and create entity references
for (Iterator<JsonNode> it =
((ArrayNode) JsonUtils.extractValue(json, "hits", "hits")).elements();
it.hasNext(); ) {
JsonNode jsonNode = it.next();
String id = JsonUtils.extractValue(jsonNode, "_source", "id");
String fqn = JsonUtils.extractValue(jsonNode, "_source", "fullyQualifiedName");
String type = JsonUtils.extractValue(jsonNode, "_source", "entityType");
if (!CommonUtil.nullOrEmpty(fqn) && !CommonUtil.nullOrEmpty(type)) {
fqns.add(
new EntityReference()
.withId(UUID.fromString(id))
.withFullyQualifiedName(fqn)
.withType(type));
}
}
// Collect the results into a map by the hash of the FQN
return fqns.stream()
.collect(
Collectors.toMap(
entityReference ->
FullyQualifiedName.buildHash(entityReference.getFullyQualifiedName()),
entityReference -> entityReference));
} catch (Exception ex) {
LOG.error("Error while fetching glossary terms with prefix from ES", ex);
}
return new HashMap<>();
}
public BulkOperationResult bulkRemoveGlossaryToAssets(
UUID glossaryTermId, AddGlossaryToAssetsRequest request) {
GlossaryTerm term = this.get(null, glossaryTermId, getFields("id,tags"));
@ -532,6 +585,10 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
closeApprovalTask(updated, "Rejected the glossary term");
}
}
if (!nullOrEmpty(original)
&& !original.getFullyQualifiedName().equals(updated.getFullyQualifiedName())) {
updateAssetIndexesOnGlossaryTermUpdate(original, updated);
}
}
@Override
@ -678,6 +735,42 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
}
}
private void updateAssetIndexesOnGlossaryTermUpdate(GlossaryTerm original, GlossaryTerm updated) {
// Update ES indexes of entity tagged with the glossary term and its children terms to reflect
// its latest value.
Set<String> targetFQNHashesFromDb =
new HashSet<>(
daoCollection.tagUsageDAO().getTargetFQNHashForTag(updated.getFullyQualifiedName()));
List<EntityReference> childTerms =
super.getChildren(updated); // get new value of children terms from DB
for (EntityReference child : childTerms) {
targetFQNHashesFromDb.addAll( // for each child term find the targetFQNHashes of assets
daoCollection.tagUsageDAO().getTargetFQNHashForTag(child.getFullyQualifiedName()));
}
// List of entity references tagged with the glossary term
Map<String, EntityReference> targetFQNFromES =
getGlossaryUsageFromES(original.getFullyQualifiedName(), targetFQNHashesFromDb.size());
Map<String, EntityReference> childrenTerms =
getGlossaryTermsContainingFQNFromES(
original.getFullyQualifiedName(),
getChildrenCount(updated)); // get old value of children term from ES
for (EntityReference child : childrenTerms.values()) {
targetFQNFromES.putAll( // List of entity references tagged with the children term
getGlossaryUsageFromES(child.getFullyQualifiedName(), targetFQNHashesFromDb.size()));
searchRepository.updateEntity(child); // update es index of child term
}
if (targetFQNFromES.size() == targetFQNHashesFromDb.size()) {
for (String fqnHash : targetFQNHashesFromDb) {
EntityReference refDetails = targetFQNFromES.get(fqnHash);
searchRepository.updateEntity(refDetails); // update ES index of assets
}
}
}
/** Handles entity updated from PUT and POST operation. */
public class GlossaryTermUpdater extends EntityUpdater {
public GlossaryTermUpdater(GlossaryTerm original, GlossaryTerm updated, Operation operation) {
@ -844,6 +937,14 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
TagSource.GLOSSARY.ordinal(),
original.getFullyQualifiedName(),
updated.getFullyQualifiedName());
daoCollection
.tagUsageDAO()
.renameByTargetFQNHash(
TagSource.CLASSIFICATION.ordinal(),
original.getFullyQualifiedName(),
updated.getFullyQualifiedName());
if (glossaryChanged) {
updateGlossaryRelationship(original, updated);
recordChange(

62
openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java
View File

@ -349,6 +349,47 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
EntityUpdater.setSessionTimeout(10 * 60 * 1000); // Turn consolidation of changes back on
}
@Test
void test_patch_changeParent_UpdateHierarchy(TestInfo test) throws IOException {
CreateGlossary create = createRequest(getEntityName(test), "", "", null);
Glossary glossary = createEntity(create, ADMIN_AUTH_HEADERS);
//
// These test move a glossary term to different parts of the glossary hierarchy and to different
// glossaries
//
// Create glossary with the following hierarchy
// -> t1 -> t11
// -> t2
// Create a Classification with the same name as glossary and assign it to a table
ClassificationResourceTest classificationResourceTest = new ClassificationResourceTest();
TagResourceTest tagResourceTest = new TagResourceTest();
CreateClassification createClassification =
classificationResourceTest.createRequest("SampleTags");
classificationResourceTest.createEntity(createClassification, ADMIN_AUTH_HEADERS);
Tag tag1 = tagResourceTest.createTag("tag1", "SampleTags", null);
Tag tag2 = tagResourceTest.createTag("tag2", "SampleTags", null);
GlossaryTermResourceTest glossaryTermResourceTest = new GlossaryTermResourceTest();
GlossaryTerm t1 = createGlossaryTerm(glossaryTermResourceTest, glossary, null, "parentTerm1");
// GlossaryTerm t11 = createGlossaryTerm(glossaryTermResourceTest, glossary, t1,
// "parentTerm11").withTags(toTagLabels(tag1,tag2));
GlossaryTerm t11 =
createGlossaryTermWithTags(
glossaryTermResourceTest, glossary, t1, "parentTerm11", toTagLabels(tag1, tag2));
GlossaryTerm originalT1 = new GlossaryTerm();
copyGlossaryTerm(t11, originalT1);
GlossaryTerm t2 = createGlossaryTerm(glossaryTermResourceTest, glossary, null, "parentTerm2");
LOG.info(" t11 == {}", t11.getTags());
LOG.info(" originalT1 == {}", originalT1.getTags());
glossaryTermResourceTest.moveGlossaryTerm(
glossary.getEntityReference(), t2.getEntityReference(), t11);
TestUtils.validateTags(originalT1.getTags(), t11.getTags());
}
@Test
void patch_moveGlossaryTermParentToChild() {}
@ -447,7 +488,8 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
.withParent(from.getParent())
.withFullyQualifiedName(from.getFullyQualifiedName())
.withChangeDescription(from.getChangeDescription())
.withVersion(from.getVersion());
.withVersion(from.getVersion())
.withTags(from.getTags());
}
@Override
@ -517,6 +559,24 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
return resource.createEntity(create, ADMIN_AUTH_HEADERS);
}
private GlossaryTerm createGlossaryTermWithTags(
GlossaryTermResourceTest resource,
Glossary glossary,
GlossaryTerm parent,
String name,
List<TagLabel> tags)
throws HttpResponseException {
CreateGlossaryTerm create =
new CreateGlossaryTerm()
.withName(name)
.withDescription("d")
.withGlossary(glossary.getFullyQualifiedName())
.withParent(getFqn(parent))
.withProvider(ProviderType.USER)
.withTags(tags);
return resource.createEntity(create, ADMIN_AUTH_HEADERS);
}
public void renameGlossaryAndCheck(Glossary glossary, String newName) throws IOException {
String oldName = glossary.getName();
String json = JsonUtils.pojoToJson(glossary);

9
openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.ts
View File

@ -1260,12 +1260,9 @@ describe('Glossary page should work properly', { tags: 'Governance' }, () => {
verifyResponseStatusCode('@saveGlossaryTermData', 200);
verifyResponseStatusCode('@fetchGlossaryTermData', 200);
/**
* Todo: Enable this once this asset issue is resolve https://github.com/open-metadata/OpenMetadata/issues/15809
*/
// cy.get('[data-testid="assets"] [data-testid="filter-count"]')
// .should('be.visible')
// .contains('3');
cy.get('[data-testid="assets"] [data-testid="filter-count"]')
.should('be.visible')
.contains('3');
// checking the breadcrumb, if the change parent term is updated and displayed
cy.get('[data-testid="breadcrumb-link"]')