yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-20 05:15:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #9721 Add support for count of terms in Glossary and Classification (#9733)

Browse Source
This commit is contained in:
Suresh Srinivas 2023-01-13 11:40:24 -08:00 committed by GitHub
parent b0ace9446d
commit b6e7bcfee5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 154 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationRepository.java
View File

@ -51,6 +51,7 @@ public class ClassificationRepository extends EntityRepository<Classification> {
@Override
public Classification setFields(Classification category, Fields fields) {
category.withTermCount(fields.contains("termCount") ? getTermCount(category) : null);
return category.withUsageCount(fields.contains("usageCount") ? getUsageCount(category) : null);
}
@ -67,6 +68,11 @@ public class ClassificationRepository extends EntityRepository<Classification> {
@Override
public void storeRelationships(Classification entity) {}
private int getTermCount(Classification category) {
ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parent", category.getName());
return daoCollection.tagDAO().listCount(filter);
}
private Integer getUsageCount(Classification category) {
return daoCollection.tagUsageDAO().getTagCount(TagSource.TAG.ordinal(), category.getName());
}

6
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java
View File

@ -72,6 +72,7 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
@Override
public Glossary setFields(Glossary glossary, Fields fields) throws IOException {
glossary.setTermCount(fields.contains("termCount") ? getTermCount(glossary) : null);
glossary.setReviewers(fields.contains("reviewers") ? getReviewers(glossary) : null);
return glossary.withUsageCount(fields.contains("usageCount") ? getUsageCount(glossary) : null);
}
@ -110,6 +111,11 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
return daoCollection.tagUsageDAO().getTagCount(TagSource.GLOSSARY.ordinal(), glossary.getName());
}
private Integer getTermCount(Glossary glossary) {
ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parent", glossary.getName());
return daoCollection.glossaryTermDAO().listCount(filter);
}
@Override
public EntityUpdater getUpdater(Glossary original, Glossary updated, Operation operation) {
return new GlossaryUpdater(original, updated, operation);

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java
View File

@ -90,7 +90,7 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
}
}
static final String FIELDS = "owner,tags,reviewers,usageCount";
static final String FIELDS = "owner,tags,reviewers,usageCount,termCount";
@GET
@Valid

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java
View File

@ -80,7 +80,7 @@ public class ClassificationResource extends EntityResource<Classification, Class
}
@SuppressWarnings("unused") // Method used by reflection
static final String FIELDS = "usageCount";
static final String FIELDS = "usageCount,termCount";
@GET
@Operation(

6
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -1736,7 +1736,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
}
/** Helper function to create an entity, submit POST API request and validate response. */
public final T createAndCheckEntity(K create, Map<String, String> authHeaders) throws IOException {
public T createAndCheckEntity(K create, Map<String, String> authHeaders) throws IOException {
// Validate an entity that is created has all the information set in create request
String updatedBy = SecurityUtil.getPrincipalName(authHeaders);
T entity = createEntity(create, authHeaders);
@ -1762,7 +1762,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
return entity;
}
public final T updateAndCheckEntity(
public T updateAndCheckEntity(
K request,
Status status,
Map<String, String> authHeaders,
@ -2331,7 +2331,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
return ref != null ? new EntityReference().withType(ref.getType()).withId(ref.getId()) : null;
}
protected String getAllowedFields() {
public String getAllowedFields() {
return String.join(",", Entity.getAllowedFields(entityClass));
}
}

66
openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java
View File

@ -51,6 +51,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import javax.ws.rs.core.Response;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
@ -81,6 +82,8 @@ import org.openmetadata.service.util.TestUtils.UpdateType;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, CreateGlossaryTerm> {
private final GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest();
public GlossaryTermResourceTest() {
super(
Entity.GLOSSARY_TERM,
@ -99,9 +102,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
// - term1
// - term11
// - term12
GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest();
CreateGlossary createGlossary = glossaryResourceTest.createRequest("glossary1", "", "", null);
Glossary glossary1 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
Glossary glossary1 = createGlossary("glossary1");
GlossaryTerm term1 = createTerm(glossary1, null, "term1");
GlossaryTerm term11 = createTerm(glossary1, term1, "term11");
@ -113,8 +114,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
// - term2
// - term21
// - term22
createGlossary = glossaryResourceTest.createRequest("glossary2", "", "", null);
Glossary glossary2 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
Glossary glossary2 = createGlossary("glossary2");
GlossaryTerm term2 = createTerm(glossary2, null, "term2");
GlossaryTerm term21 = createTerm(glossary2, term2, "term21");
@ -166,17 +166,14 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
//
// When reviewers are not set for a glossary term, carry it forward from the glossary
//
GlossaryResourceTest glossaryTest = new GlossaryResourceTest();
CreateGlossary create =
glossaryTest.createRequest(getEntityName(test)).withReviewers(List.of(USER1_REF)).withDescription("d");
Glossary glossary = glossaryTest.createEntity(create, ADMIN_AUTH_HEADERS);
Glossary glossary = createGlossary(test);
// Create terms t1 and a term t12 under t1 in the glossary without reviewers
GlossaryTerm t1 = createTerm(glossary, null, "t1", null);
assertEquals(create.getReviewers(), t1.getReviewers()); // Reviewers are inherited
assertEquals(glossary.getReviewers(), t1.getReviewers()); // Reviewers are inherited
GlossaryTerm t12 = createTerm(glossary, t1, "t12", null);
assertEquals(create.getReviewers(), t12.getReviewers()); // Reviewers are inherited
assertEquals(glossary.getReviewers(), t12.getReviewers()); // Reviewers are inherited
}
@Test
@ -184,10 +181,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
//
// Create glossary terms that start with common prefix and make sure usage count is correct
//
GlossaryResourceTest glossaryTest = new GlossaryResourceTest();
CreateGlossary create =
glossaryTest.createRequest(getEntityName(test)).withReviewers(List.of(USER1_REF)).withDescription("d");
Glossary glossary = glossaryTest.createEntity(create, ADMIN_AUTH_HEADERS);
Glossary glossary = createGlossary(test);
// Create nested terms a -> aa -> aaa;
GlossaryTerm a = createTerm(glossary, null, "a", null);
@ -284,9 +278,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
@Test
void delete_recursive(TestInfo test) throws IOException {
GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest();
CreateGlossary createGlossary = glossaryResourceTest.createRequest(getEntityName(test), "", "", null);
Glossary g1 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
Glossary g1 = createGlossary(test);
EntityReference g1Ref = g1.getEntityReference();
// Create glossary term t1 in glossary g1
@ -500,6 +492,31 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
}
}
@Override
public GlossaryTerm createAndCheckEntity(CreateGlossaryTerm create, Map<String, String> authHeaders)
throws IOException {
int termCount = getGlossary(create.getGlossary().getName()).getTermCount();
GlossaryTerm term = super.createAndCheckEntity(create, authHeaders);
assertEquals(termCount + 1, getGlossary(create.getGlossary().getName()).getTermCount());
return term;
}
@Override
public GlossaryTerm updateAndCheckEntity(
CreateGlossaryTerm request,
Response.Status status,
Map<String, String> authHeaders,
UpdateType updateType,
ChangeDescription changeDescription)
throws IOException {
int termCount = getGlossary(request.getGlossary().getName()).getTermCount();
GlossaryTerm term = super.updateAndCheckEntity(request, status, authHeaders, updateType, changeDescription);
if (status == Response.Status.CREATED) {
assertEquals(termCount + 1, getGlossary(request.getGlossary().getName()).getTermCount());
}
return term;
}
public void renameGlossaryTermAndCheck(GlossaryTerm term, String newName) throws IOException {
String oldName = term.getName();
String json = JsonUtils.pojoToJson(term);
@ -572,4 +589,17 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
assertTrue(child.getFullyQualifiedName().startsWith(newTerm.getFullyQualifiedName()));
}
}
public Glossary createGlossary(TestInfo test) throws IOException {
return createGlossary(glossaryResourceTest.getEntityName(test));
}
public Glossary createGlossary(String name) throws IOException {
CreateGlossary create = glossaryResourceTest.createRequest(name);
return glossaryResourceTest.createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
}
public Glossary getGlossary(String name) throws IOException {
return glossaryResourceTest.getEntityByName(name, glossaryResourceTest.getAllowedFields(), ADMIN_AUTH_HEADERS);
}
}

1
openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/ClassificationResourceTest.java
View File

@ -133,6 +133,7 @@ public class ClassificationResourceTest extends EntityResourceTest<Classificatio
}
public void renameClassificationAndCheck(Classification classification, String newName) throws IOException {
// User PATCH operation to rename a classification
String oldName = classification.getName();
String json = JsonUtils.pojoToJson(classification);
ChangeDescription change = getChangeDescription(classification.getVersion());

113
openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/TagResourceTest.java
View File

@ -19,21 +19,22 @@ import static javax.ws.rs.core.Response.Status.NOT_FOUND;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.common.utils.CommonUtil.listOf;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound;
import static org.openmetadata.service.security.SecurityUtil.authHeaders;
import static org.openmetadata.service.util.EntityUtil.fieldUpdated;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.service.util.TestUtils.UpdateType.NO_CHANGE;
import static org.openmetadata.service.util.TestUtils.assertListNotNull;
import static org.openmetadata.service.util.TestUtils.assertListNull;
import static org.openmetadata.service.util.TestUtils.assertResponse;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpResponseException;
@ -57,6 +58,7 @@ import org.openmetadata.service.resources.tags.TagResource.TagList;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.FullyQualifiedName;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.TestUtils.UpdateType;
/** Tests not covered here: Classification and Tag usage counts are covered in TableResourceTest */
@Slf4j
@ -75,16 +77,15 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
TIER1_TAG_LABEL = getTagLabel(FullyQualifiedName.add("Tier", "Tier1"));
TIER2_TAG_LABEL = getTagLabel(FullyQualifiedName.add("Tier", "Tier2"));
CreateClassification create = classificationResourceTest.createRequest("User");
USER_TAG_CATEGORY = classificationResourceTest.createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
USER_TAG_CATEGORY = createClassification("User");
List<String> associatedTags = new ArrayList<>();
associatedTags.add(PERSONAL_DATA_TAG_LABEL.getTagFQN());
associatedTags.add(PII_SENSITIVE_TAG_LABEL.getTagFQN());
CreateTag createTag =
createRequest("Address").withClassification(USER_TAG_CATEGORY.getName()).withAssociatedTags(associatedTags);
ADDRESS_TAG = createEntity(createTag, ADMIN_AUTH_HEADERS);
ADDRESS_TAG =
createTag(
"Address",
USER_TAG_CATEGORY.getName(),
null,
PERSONAL_DATA_TAG_LABEL.getTagFQN(),
PII_SENSITIVE_TAG_LABEL.getTagFQN());
USER_ADDRESS_TAG_LABEL = getTagLabel(FullyQualifiedName.add("User", "Address"));
}
@ -94,31 +95,27 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
@Order(1)
@Test
void post_validTags_200() throws HttpResponseException {
Classification category =
classificationResourceTest.getEntityByName(USER_TAG_CATEGORY.getName(), authHeaders("test@open-meatadata.org"));
void post_validTags_200() throws IOException {
Classification classification = getClassification(USER_TAG_CATEGORY.getName());
Map<String, String> queryParams = new HashMap<>();
queryParams.put("parent", category.getFullyQualifiedName());
queryParams.put("parent", classification.getFullyQualifiedName());
List<Tag> childrenBefore = listEntities(queryParams, ADMIN_AUTH_HEADERS).getData();
CreateTag create = createRequest("tag1").withClassification(category.getName());
Tag tag1 = createEntity(create, ADMIN_AUTH_HEADERS);
Tag tag1 = createTag("tag1", classification.getName(), null);
List<Tag> childrenAfter = listEntities(queryParams, ADMIN_AUTH_HEADERS).getData();
assertEquals(childrenBefore.size() + 1, childrenAfter.size());
// POST .../tags/{category}/{primaryTag}/{secondaryTag} to create secondary tag
create = createRequest("SecondaryTag").withParent(tag1.getFullyQualifiedName());
createEntity(create, ADMIN_AUTH_HEADERS);
createTag("SecondaryTag", classification.getName(), tag1.getFullyQualifiedName());
}
@Test
void post_newTagsOnNonExistentParents_404() {
// POST .../tags/{nonExistent}/{primaryTag} where category does not exist
String nonExistent = "nonExistent";
CreateTag create = createRequest("primary").withClassification(nonExistent);
assertResponse(
() -> createEntity(create, ADMIN_AUTH_HEADERS), NOT_FOUND, entityNotFound(Entity.CLASSIFICATION, nonExistent));
() -> createTag("primary", nonExistent, null), NOT_FOUND, entityNotFound(Entity.CLASSIFICATION, nonExistent));
// POST .../tags/{user}/{nonExistent}/tag where primaryTag does not exist
String parentFqn = FullyQualifiedName.build(USER_TAG_CATEGORY.getName(), nonExistent);
@ -133,17 +130,15 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
// Create under tag1 secondary tags t11 t12
// Create under tag2 secondary tags t21 t22
//
String categoryName = test.getDisplayName().substring(0, 10);
CreateClassification createCategory = classificationResourceTest.createRequest(categoryName);
Classification category =
classificationResourceTest.updateEntity(createCategory, Status.CREATED, ADMIN_AUTH_HEADERS);
String classificationName = test.getDisplayName().substring(0, 10);
Classification classification = createClassification(classificationName);
Tag t1 = createOrUpdate(categoryName, null, "t1", CREATED);
createOrUpdate(categoryName, t1, "t11", CREATED);
createOrUpdate(categoryName, t1, "t12", CREATED);
Tag t2 = createOrUpdate(categoryName, null, "t2", CREATED);
createOrUpdate(categoryName, t2, "t21", CREATED);
Tag t22 = createOrUpdate(categoryName, t2, "t22", CREATED);
Tag t1 = createOrUpdate(classificationName, null, "t1", CREATED);
createOrUpdate(classificationName, t1, "t11", CREATED);
createOrUpdate(classificationName, t1, "t12", CREATED);
Tag t2 = createOrUpdate(classificationName, null, "t2", CREATED);
createOrUpdate(classificationName, t2, "t21", CREATED);
Tag t22 = createOrUpdate(classificationName, t2, "t22", CREATED);
// Rename leaf node t22 to newt22
renameTagAndCheck(t22, "newt22");
@ -152,8 +147,8 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
renameTagAndCheck(t2, "newt2");
// Change classification name and ensure all the tags have the new names
String newCategoryName = "new" + categoryName;
classificationResourceTest.renameClassificationAndCheck(category, newCategoryName);
String newclassificationName = "new" + classificationName;
classificationResourceTest.renameClassificationAndCheck(classification, newclassificationName);
}
@Test
@ -165,11 +160,12 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
CatalogExceptionMessage.systemEntityDeleteNotAllowed(tag.getName(), Entity.TAG));
}
private Tag createOrUpdate(String categoryName, Tag parent, String name, Status status) throws HttpResponseException {
private Tag createOrUpdate(String classificationName, Tag parent, String name, Status status) throws IOException {
String parentFqn = parent != null ? parent.getFullyQualifiedName() : null;
CreateTag createTag =
createRequest(name).withParent(parentFqn).withClassification(categoryName).withDescription("description");
return updateEntity(createTag, status, ADMIN_AUTH_HEADERS); // Change to updateAndCheck
createRequest(name).withParent(parentFqn).withClassification(classificationName).withDescription("description");
Tag tag = updateAndCheckEntity(createTag, status, ADMIN_AUTH_HEADERS, NO_CHANGE, null);
return tag;
}
public void renameTagAndCheck(Tag tag, String newName) throws IOException {
@ -250,4 +246,49 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
}
assertCommonFieldChange(fieldName, expected, actual);
}
@Override
public Tag createAndCheckEntity(CreateTag create, Map<String, String> authHeaders) throws IOException {
int termCount = getClassification(create.getClassification()).getTermCount();
Tag tag = super.createAndCheckEntity(create, authHeaders);
assertEquals(termCount + 1, getClassification(create.getClassification()).getTermCount());
return tag;
}
@Override
public Tag updateAndCheckEntity(
CreateTag request,
Status status,
Map<String, String> authHeaders,
UpdateType updateType,
ChangeDescription changeDescription)
throws IOException {
int termCount = getClassification(request.getClassification()).getTermCount();
Tag tag = super.updateAndCheckEntity(request, status, authHeaders, updateType, changeDescription);
if (status == Response.Status.CREATED) {
assertEquals(termCount + 1, getClassification(request.getClassification()).getTermCount());
}
return tag;
}
public Tag createTag(String name, String classification, String parentFqn, String... associatedTags)
throws IOException {
List<String> associatedTagList = associatedTags.length == 0 ? null : listOf(associatedTags);
CreateTag createTag =
createRequest(name)
.withParent(parentFqn)
.withClassification(classification)
.withAssociatedTags(associatedTagList);
return createEntity(createTag, ADMIN_AUTH_HEADERS);
}
public Classification createClassification(String name) throws IOException {
CreateClassification create = classificationResourceTest.createRequest(name);
return classificationResourceTest.createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
}
public Classification getClassification(String name) throws IOException {
return classificationResourceTest.getEntityByName(
name, classificationResourceTest.getAllowedFields(), ADMIN_AUTH_HEADERS);
}
}

2
openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java
View File

@ -1089,7 +1089,7 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
}
@Override
protected String getAllowedFields() {
public String getAllowedFields() {
List<String> allowedFields = Entity.getAllowedFields(entityClass);
allowedFields.removeAll(of(USER_PROTECTED_FIELDS.split(",")));
return String.join(",", allowedFields);

5
openmetadata-spec/src/main/resources/json/schema/entity/classification/classification.json
View File

@ -30,6 +30,11 @@
"description": "Metadata version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/entityVersion"
},
"termCount" : {
"description": "Total number of children tag terms under this classification. This includes all the children in the hierarchy.",
"type" : "integer",
"minimum": 0
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../../type/basic.json#/definitions/timestamp"

5
openmetadata-spec/src/main/resources/json/schema/entity/data/glossary.json
View File

@ -73,6 +73,11 @@
},
"default": null
},
"termCount" : {
"description": "Total number of terms in the glossary. This includes all the children in the hierarchy.",
"type" : "integer",
"minimum": 0
},
"changeDescription": {
"description": "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"