Fixes #9721 Add support for count of terms in Glossary and Classification (#9733)

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationRepository.java

@@ -51,6 +51,7 @@ public class ClassificationRepository extends EntityRepository<Classification> {
 
   @Override
   public Classification setFields(Classification category, Fields fields) {
+    category.withTermCount(fields.contains("termCount") ? getTermCount(category) : null);
     return category.withUsageCount(fields.contains("usageCount") ? getUsageCount(category) : null);
   }
 
@@ -67,6 +68,11 @@ public class ClassificationRepository extends EntityRepository<Classification> {
   @Override
   public void storeRelationships(Classification entity) {}
 
+  private int getTermCount(Classification category) {
+    ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parent", category.getName());
+    return daoCollection.tagDAO().listCount(filter);
+  }
+
   private Integer getUsageCount(Classification category) {
     return daoCollection.tagUsageDAO().getTagCount(TagSource.TAG.ordinal(), category.getName());
   }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java

@@ -72,6 +72,7 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
 
   @Override
   public Glossary setFields(Glossary glossary, Fields fields) throws IOException {
+    glossary.setTermCount(fields.contains("termCount") ? getTermCount(glossary) : null);
     glossary.setReviewers(fields.contains("reviewers") ? getReviewers(glossary) : null);
     return glossary.withUsageCount(fields.contains("usageCount") ? getUsageCount(glossary) : null);
   }
@@ -110,6 +111,11 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
     return daoCollection.tagUsageDAO().getTagCount(TagSource.GLOSSARY.ordinal(), glossary.getName());
   }
 
+  private Integer getTermCount(Glossary glossary) {
+    ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parent", glossary.getName());
+    return daoCollection.glossaryTermDAO().listCount(filter);
+  }
+
   @Override
   public EntityUpdater getUpdater(Glossary original, Glossary updated, Operation operation) {
     return new GlossaryUpdater(original, updated, operation);

openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java

@@ -90,7 +90,7 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
     }
   }
 
-  static final String FIELDS = "owner,tags,reviewers,usageCount";
+  static final String FIELDS = "owner,tags,reviewers,usageCount,termCount";
 
   @GET
   @Valid

openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java

@@ -80,7 +80,7 @@ public class ClassificationResource extends EntityResource<Classification, Class
   }
 
   @SuppressWarnings("unused") // Method used by reflection
-  static final String FIELDS = "usageCount";
+  static final String FIELDS = "usageCount,termCount";
 
   @GET
   @Operation(

openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java

@@ -1736,7 +1736,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
   }
 
   /** Helper function to create an entity, submit POST API request and validate response. */
-  public final T createAndCheckEntity(K create, Map<String, String> authHeaders) throws IOException {
+  public T createAndCheckEntity(K create, Map<String, String> authHeaders) throws IOException {
     // Validate an entity that is created has all the information set in create request
     String updatedBy = SecurityUtil.getPrincipalName(authHeaders);
     T entity = createEntity(create, authHeaders);
@@ -1762,7 +1762,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
     return entity;
   }
 
-  public final T updateAndCheckEntity(
+  public T updateAndCheckEntity(
       K request,
       Status status,
       Map<String, String> authHeaders,
@@ -2331,7 +2331,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
     return ref != null ? new EntityReference().withType(ref.getType()).withId(ref.getId()) : null;
   }
 
-  protected String getAllowedFields() {
+  public String getAllowedFields() {
     return String.join(",", Entity.getAllowedFields(entityClass));
   }
 }

openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java

@@ -51,6 +51,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import javax.ws.rs.core.Response;
 import org.apache.http.client.HttpResponseException;
 import org.junit.jupiter.api.MethodOrderer;
 import org.junit.jupiter.api.Order;
@@ -81,6 +82,8 @@ import org.openmetadata.service.util.TestUtils.UpdateType;
 
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
 public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, CreateGlossaryTerm> {
+  private final GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest();
+
   public GlossaryTermResourceTest() {
     super(
         Entity.GLOSSARY_TERM,
@@ -99,9 +102,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
     // - term1
     //   - term11
     //   - term12
-    GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest();
+    Glossary glossary1 = createGlossary("glossary1");
-    CreateGlossary createGlossary = glossaryResourceTest.createRequest("glossary1", "", "", null);
-    Glossary glossary1 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
 
     GlossaryTerm term1 = createTerm(glossary1, null, "term1");
     GlossaryTerm term11 = createTerm(glossary1, term1, "term11");
@@ -113,8 +114,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
     // - term2
     //   - term21
     //   - term22
-    createGlossary = glossaryResourceTest.createRequest("glossary2", "", "", null);
+    Glossary glossary2 = createGlossary("glossary2");
-    Glossary glossary2 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
 
     GlossaryTerm term2 = createTerm(glossary2, null, "term2");
     GlossaryTerm term21 = createTerm(glossary2, term2, "term21");
@@ -166,17 +166,14 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
     //
     // When reviewers are not set for a glossary term, carry it forward from the glossary
     //
-    GlossaryResourceTest glossaryTest = new GlossaryResourceTest();
+    Glossary glossary = createGlossary(test);
-    CreateGlossary create =
-        glossaryTest.createRequest(getEntityName(test)).withReviewers(List.of(USER1_REF)).withDescription("d");
-    Glossary glossary = glossaryTest.createEntity(create, ADMIN_AUTH_HEADERS);
 
     // Create terms t1 and a term t12 under t1 in the glossary without reviewers
     GlossaryTerm t1 = createTerm(glossary, null, "t1", null);
-    assertEquals(create.getReviewers(), t1.getReviewers()); // Reviewers are inherited
+    assertEquals(glossary.getReviewers(), t1.getReviewers()); // Reviewers are inherited
 
     GlossaryTerm t12 = createTerm(glossary, t1, "t12", null);
-    assertEquals(create.getReviewers(), t12.getReviewers()); // Reviewers are inherited
+    assertEquals(glossary.getReviewers(), t12.getReviewers()); // Reviewers are inherited
   }
 
   @Test
@@ -184,10 +181,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
     //
     // Create glossary terms that start with common prefix and make sure usage count is correct
     //
-    GlossaryResourceTest glossaryTest = new GlossaryResourceTest();
+    Glossary glossary = createGlossary(test);
-    CreateGlossary create =
-        glossaryTest.createRequest(getEntityName(test)).withReviewers(List.of(USER1_REF)).withDescription("d");
-    Glossary glossary = glossaryTest.createEntity(create, ADMIN_AUTH_HEADERS);
 
     // Create nested terms a -> aa -> aaa;
     GlossaryTerm a = createTerm(glossary, null, "a", null);
@@ -284,9 +278,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
 
   @Test
   void delete_recursive(TestInfo test) throws IOException {
-    GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest();
+    Glossary g1 = createGlossary(test);
-    CreateGlossary createGlossary = glossaryResourceTest.createRequest(getEntityName(test), "", "", null);
-    Glossary g1 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
     EntityReference g1Ref = g1.getEntityReference();
 
     // Create glossary term t1 in glossary g1
@@ -500,6 +492,31 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
     }
   }
 
+  @Override
+  public GlossaryTerm createAndCheckEntity(CreateGlossaryTerm create, Map<String, String> authHeaders)
+      throws IOException {
+    int termCount = getGlossary(create.getGlossary().getName()).getTermCount();
+    GlossaryTerm term = super.createAndCheckEntity(create, authHeaders);
+    assertEquals(termCount + 1, getGlossary(create.getGlossary().getName()).getTermCount());
+    return term;
+  }
+
+  @Override
+  public GlossaryTerm updateAndCheckEntity(
+      CreateGlossaryTerm request,
+      Response.Status status,
+      Map<String, String> authHeaders,
+      UpdateType updateType,
+      ChangeDescription changeDescription)
+      throws IOException {
+    int termCount = getGlossary(request.getGlossary().getName()).getTermCount();
+    GlossaryTerm term = super.updateAndCheckEntity(request, status, authHeaders, updateType, changeDescription);
+    if (status == Response.Status.CREATED) {
+      assertEquals(termCount + 1, getGlossary(request.getGlossary().getName()).getTermCount());
+    }
+    return term;
+  }
+
   public void renameGlossaryTermAndCheck(GlossaryTerm term, String newName) throws IOException {
     String oldName = term.getName();
     String json = JsonUtils.pojoToJson(term);
@@ -572,4 +589,17 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
       assertTrue(child.getFullyQualifiedName().startsWith(newTerm.getFullyQualifiedName()));
     }
   }
+
+  public Glossary createGlossary(TestInfo test) throws IOException {
+    return createGlossary(glossaryResourceTest.getEntityName(test));
+  }
+
+  public Glossary createGlossary(String name) throws IOException {
+    CreateGlossary create = glossaryResourceTest.createRequest(name);
+    return glossaryResourceTest.createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
+  }
+
+  public Glossary getGlossary(String name) throws IOException {
+    return glossaryResourceTest.getEntityByName(name, glossaryResourceTest.getAllowedFields(), ADMIN_AUTH_HEADERS);
+  }
 }

openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/ClassificationResourceTest.java

@@ -133,6 +133,7 @@ public class ClassificationResourceTest extends EntityResourceTest<Classificatio
   }
 
   public void renameClassificationAndCheck(Classification classification, String newName) throws IOException {
+    // User PATCH operation to rename a classification
     String oldName = classification.getName();
     String json = JsonUtils.pojoToJson(classification);
     ChangeDescription change = getChangeDescription(classification.getVersion());

openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/TagResourceTest.java

@@ -19,21 +19,22 @@ import static javax.ws.rs.core.Response.Status.NOT_FOUND;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.openmetadata.common.utils.CommonUtil.listOf;
 import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
 import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound;
-import static org.openmetadata.service.security.SecurityUtil.authHeaders;
 import static org.openmetadata.service.util.EntityUtil.fieldUpdated;
 import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
 import static org.openmetadata.service.util.TestUtils.UpdateType.MINOR_UPDATE;
+import static org.openmetadata.service.util.TestUtils.UpdateType.NO_CHANGE;
 import static org.openmetadata.service.util.TestUtils.assertListNotNull;
 import static org.openmetadata.service.util.TestUtils.assertListNull;
 import static org.openmetadata.service.util.TestUtils.assertResponse;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.client.HttpResponseException;
@@ -57,6 +58,7 @@ import org.openmetadata.service.resources.tags.TagResource.TagList;
 import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.FullyQualifiedName;
 import org.openmetadata.service.util.JsonUtils;
+import org.openmetadata.service.util.TestUtils.UpdateType;
 
 /** Tests not covered here: Classification and Tag usage counts are covered in TableResourceTest */
 @Slf4j
@@ -75,16 +77,15 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
     TIER1_TAG_LABEL = getTagLabel(FullyQualifiedName.add("Tier", "Tier1"));
     TIER2_TAG_LABEL = getTagLabel(FullyQualifiedName.add("Tier", "Tier2"));
 
-    CreateClassification create = classificationResourceTest.createRequest("User");
+    USER_TAG_CATEGORY = createClassification("User");
-    USER_TAG_CATEGORY = classificationResourceTest.createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
 
-    List<String> associatedTags = new ArrayList<>();
+    ADDRESS_TAG =
-    associatedTags.add(PERSONAL_DATA_TAG_LABEL.getTagFQN());
+        createTag(
-    associatedTags.add(PII_SENSITIVE_TAG_LABEL.getTagFQN());
+            "Address",
-
+            USER_TAG_CATEGORY.getName(),
-    CreateTag createTag =
+            null,
-        createRequest("Address").withClassification(USER_TAG_CATEGORY.getName()).withAssociatedTags(associatedTags);
+            PERSONAL_DATA_TAG_LABEL.getTagFQN(),
-    ADDRESS_TAG = createEntity(createTag, ADMIN_AUTH_HEADERS);
+            PII_SENSITIVE_TAG_LABEL.getTagFQN());
     USER_ADDRESS_TAG_LABEL = getTagLabel(FullyQualifiedName.add("User", "Address"));
   }
 
@@ -94,31 +95,27 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
 
   @Order(1)
   @Test
-  void post_validTags_200() throws HttpResponseException {
+  void post_validTags_200() throws IOException {
-    Classification category =
+    Classification classification = getClassification(USER_TAG_CATEGORY.getName());
-        classificationResourceTest.getEntityByName(USER_TAG_CATEGORY.getName(), authHeaders("test@open-meatadata.org"));
     Map<String, String> queryParams = new HashMap<>();
-    queryParams.put("parent", category.getFullyQualifiedName());
+    queryParams.put("parent", classification.getFullyQualifiedName());
     List<Tag> childrenBefore = listEntities(queryParams, ADMIN_AUTH_HEADERS).getData();
 
-    CreateTag create = createRequest("tag1").withClassification(category.getName());
+    Tag tag1 = createTag("tag1", classification.getName(), null);
-    Tag tag1 = createEntity(create, ADMIN_AUTH_HEADERS);
     List<Tag> childrenAfter = listEntities(queryParams, ADMIN_AUTH_HEADERS).getData();
 
     assertEquals(childrenBefore.size() + 1, childrenAfter.size());
 
     // POST .../tags/{category}/{primaryTag}/{secondaryTag} to create secondary tag
-    create = createRequest("SecondaryTag").withParent(tag1.getFullyQualifiedName());
+    createTag("SecondaryTag", classification.getName(), tag1.getFullyQualifiedName());
-    createEntity(create, ADMIN_AUTH_HEADERS);
   }
 
   @Test
   void post_newTagsOnNonExistentParents_404() {
     // POST .../tags/{nonExistent}/{primaryTag} where category does not exist
     String nonExistent = "nonExistent";
-    CreateTag create = createRequest("primary").withClassification(nonExistent);
     assertResponse(
-        () -> createEntity(create, ADMIN_AUTH_HEADERS), NOT_FOUND, entityNotFound(Entity.CLASSIFICATION, nonExistent));
+        () -> createTag("primary", nonExistent, null), NOT_FOUND, entityNotFound(Entity.CLASSIFICATION, nonExistent));
 
     // POST .../tags/{user}/{nonExistent}/tag where primaryTag does not exist
     String parentFqn = FullyQualifiedName.build(USER_TAG_CATEGORY.getName(), nonExistent);
@@ -133,17 +130,15 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
     // Create under tag1 secondary tags t11 t12
     // Create under tag2 secondary tags t21 t22
     //
-    String categoryName = test.getDisplayName().substring(0, 10);
+    String classificationName = test.getDisplayName().substring(0, 10);
-    CreateClassification createCategory = classificationResourceTest.createRequest(categoryName);
+    Classification classification = createClassification(classificationName);
-    Classification category =
-        classificationResourceTest.updateEntity(createCategory, Status.CREATED, ADMIN_AUTH_HEADERS);
 
-    Tag t1 = createOrUpdate(categoryName, null, "t1", CREATED);
+    Tag t1 = createOrUpdate(classificationName, null, "t1", CREATED);
-    createOrUpdate(categoryName, t1, "t11", CREATED);
+    createOrUpdate(classificationName, t1, "t11", CREATED);
-    createOrUpdate(categoryName, t1, "t12", CREATED);
+    createOrUpdate(classificationName, t1, "t12", CREATED);
-    Tag t2 = createOrUpdate(categoryName, null, "t2", CREATED);
+    Tag t2 = createOrUpdate(classificationName, null, "t2", CREATED);
-    createOrUpdate(categoryName, t2, "t21", CREATED);
+    createOrUpdate(classificationName, t2, "t21", CREATED);
-    Tag t22 = createOrUpdate(categoryName, t2, "t22", CREATED);
+    Tag t22 = createOrUpdate(classificationName, t2, "t22", CREATED);
 
     // Rename leaf node t22 to newt22
     renameTagAndCheck(t22, "newt22");
@@ -152,8 +147,8 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
     renameTagAndCheck(t2, "newt2");
 
     // Change classification name and ensure all the tags have the new names
-    String newCategoryName = "new" + categoryName;
+    String newclassificationName = "new" + classificationName;
-    classificationResourceTest.renameClassificationAndCheck(category, newCategoryName);
+    classificationResourceTest.renameClassificationAndCheck(classification, newclassificationName);
   }
 
   @Test
@@ -165,11 +160,12 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
         CatalogExceptionMessage.systemEntityDeleteNotAllowed(tag.getName(), Entity.TAG));
   }
 
-  private Tag createOrUpdate(String categoryName, Tag parent, String name, Status status) throws HttpResponseException {
+  private Tag createOrUpdate(String classificationName, Tag parent, String name, Status status) throws IOException {
     String parentFqn = parent != null ? parent.getFullyQualifiedName() : null;
     CreateTag createTag =
-        createRequest(name).withParent(parentFqn).withClassification(categoryName).withDescription("description");
+        createRequest(name).withParent(parentFqn).withClassification(classificationName).withDescription("description");
-    return updateEntity(createTag, status, ADMIN_AUTH_HEADERS); // Change to updateAndCheck
+    Tag tag = updateAndCheckEntity(createTag, status, ADMIN_AUTH_HEADERS, NO_CHANGE, null);
+    return tag;
   }
 
   public void renameTagAndCheck(Tag tag, String newName) throws IOException {
@@ -250,4 +246,49 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
     }
     assertCommonFieldChange(fieldName, expected, actual);
   }
+
+  @Override
+  public Tag createAndCheckEntity(CreateTag create, Map<String, String> authHeaders) throws IOException {
+    int termCount = getClassification(create.getClassification()).getTermCount();
+    Tag tag = super.createAndCheckEntity(create, authHeaders);
+    assertEquals(termCount + 1, getClassification(create.getClassification()).getTermCount());
+    return tag;
+  }
+
+  @Override
+  public Tag updateAndCheckEntity(
+      CreateTag request,
+      Status status,
+      Map<String, String> authHeaders,
+      UpdateType updateType,
+      ChangeDescription changeDescription)
+      throws IOException {
+    int termCount = getClassification(request.getClassification()).getTermCount();
+    Tag tag = super.updateAndCheckEntity(request, status, authHeaders, updateType, changeDescription);
+    if (status == Response.Status.CREATED) {
+      assertEquals(termCount + 1, getClassification(request.getClassification()).getTermCount());
+    }
+    return tag;
+  }
+
+  public Tag createTag(String name, String classification, String parentFqn, String... associatedTags)
+      throws IOException {
+    List<String> associatedTagList = associatedTags.length == 0 ? null : listOf(associatedTags);
+    CreateTag createTag =
+        createRequest(name)
+            .withParent(parentFqn)
+            .withClassification(classification)
+            .withAssociatedTags(associatedTagList);
+    return createEntity(createTag, ADMIN_AUTH_HEADERS);
+  }
+
+  public Classification createClassification(String name) throws IOException {
+    CreateClassification create = classificationResourceTest.createRequest(name);
+    return classificationResourceTest.createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
+  }
+
+  public Classification getClassification(String name) throws IOException {
+    return classificationResourceTest.getEntityByName(
+        name, classificationResourceTest.getAllowedFields(), ADMIN_AUTH_HEADERS);
+  }
 }

openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java

@@ -1089,7 +1089,7 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
   }
 
   @Override
-  protected String getAllowedFields() {
+  public String getAllowedFields() {
     List<String> allowedFields = Entity.getAllowedFields(entityClass);
     allowedFields.removeAll(of(USER_PROTECTED_FIELDS.split(",")));
     return String.join(",", allowedFields);

openmetadata-spec/src/main/resources/json/schema/entity/classification/classification.json

@@ -30,6 +30,11 @@
       "description": "Metadata version of the entity.",
       "$ref": "../../type/entityHistory.json#/definitions/entityVersion"
     },
+    "termCount" : {
+      "description": "Total number of children tag terms under this classification. This includes all the children in the hierarchy.",
+      "type" : "integer",
+      "minimum": 0
+    },
     "updatedAt": {
       "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
       "$ref": "../../type/basic.json#/definitions/timestamp"

openmetadata-spec/src/main/resources/json/schema/entity/data/glossary.json

@@ -73,6 +73,11 @@
       },
       "default": null
     },
+    "termCount" : {
+      "description": "Total number of terms in the glossary. This includes all the children in the hierarchy.",
+      "type" : "integer",
+      "minimum": 0
+    },
     "changeDescription": {
       "description": "Change that lead to this version of the entity.",
       "$ref": "../../type/entityHistory.json#/definitions/changeDescription"