From f540981de1efafdee2a23dd4987ca9a03fd196bc Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 7 Mar 2022 19:35:48 -0800 Subject: [PATCH] Fixes #3239 Add support for using glossary terms as tag labels (#3240) --- .../mysql/v004__create_db_connection_info.sql | 9 ++ .../catalog/jdbi3/ChartRepository.java | 2 +- .../catalog/jdbi3/CollectionDAO.java | 15 ++- .../catalog/jdbi3/DashboardRepository.java | 2 +- .../catalog/jdbi3/EntityRepository.java | 105 +++++++++++++++--- .../catalog/jdbi3/GlossaryRepository.java | 9 +- .../catalog/jdbi3/GlossaryTermRepository.java | 8 +- .../catalog/jdbi3/LocationRepository.java | 2 +- .../catalog/jdbi3/MetricsRepository.java | 2 +- .../catalog/jdbi3/MlModelRepository.java | 2 +- .../catalog/jdbi3/PipelineRepository.java | 2 +- .../catalog/jdbi3/TableRepository.java | 16 +-- .../openmetadata/catalog/jdbi3/TagDAO.java | 80 ------------- .../catalog/jdbi3/TagRepository.java | 5 +- .../catalog/jdbi3/TopicRepository.java | 2 +- .../resources/glossary/GlossaryResource.java | 2 +- .../glossary/GlossaryTermResource.java | 2 +- .../openmetadata/catalog/util/EntityUtil.java | 71 ------------ .../json/schema/entity/data/glossary.json | 4 + .../json/schema/entity/data/glossaryTerm.json | 4 + .../resources/json/schema/type/tagLabel.json | 8 +- .../EnumBackwardCompatibilityTest.java | 20 +++- .../catalog/resources/EntityResourceTest.java | 62 ++++++++++- .../databases/TableResourceTest.java | 101 ++++++++++------- .../glossary/GlossaryTermResourceTest.java | 39 +------ .../openmetadata/catalog/util/TestUtils.java | 4 + 26 files changed, 302 insertions(+), 276 deletions(-) delete mode 100644 catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TagDAO.java diff --git a/bootstrap/sql/mysql/v004__create_db_connection_info.sql b/bootstrap/sql/mysql/v004__create_db_connection_info.sql index 6d037ab8df9..4a18433db22 100644 --- a/bootstrap/sql/mysql/v004__create_db_connection_info.sql +++ b/bootstrap/sql/mysql/v004__create_db_connection_info.sql @@ -56,3 +56,12 @@ ALTER TABLE role_entity ADD COLUMN `default` BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.default')), ADD INDEX(`default`); +-- Add tag label source +ALTER TABLE tag_usage +ADD COLUMN source TINYINT NOT NULL FIRST, -- Source of tag (either from TagCategory or Glossary) +DROP KEY unique_name, +ADD UNIQUE KEY unique_name(source, tagFQN, targetFQN); + +UPDATE tag_usage +SET source = 0 +WHERE source IS NULL; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java index f049ac28872..5d3dcc338b2 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java @@ -67,7 +67,7 @@ public class ChartRepository extends EntityRepository { chart.setServiceType(dashboardService.getServiceType()); chart.setFullyQualifiedName(getFQN(chart)); chart.setOwner(helper(chart).validateOwnerOrNull()); - chart.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), chart.getTags())); + chart.setTags(addDerivedTags(chart.getTags())); } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java index f1c4e879473..336121ffd55 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java @@ -1050,21 +1050,23 @@ public interface CollectionDAO { String findTag(@Bind("fqn") String fqn); @SqlUpdate( - "INSERT IGNORE INTO tag_usage (tagFQN, targetFQN, labelType, state) VALUES (:tagFQN, :targetFQN, " - + ":labelType, :state)") + "INSERT IGNORE INTO tag_usage (source, tagFQN, targetFQN, labelType, state) " + + "VALUES (:source, :tagFQN, :targetFQN, :labelType, :state)") void applyTag( + @Bind("source") int source, @Bind("tagFQN") String tagFQN, @Bind("targetFQN") String targetFQN, @Bind("labelType") int labelType, @Bind("state") int state); @SqlQuery( - "SELECT tu.tagFQN, tu.labelType, tu.state, t.json ->> '$.description' AS description FROM tag_usage tu " - + "JOIN tag t ON tu.tagFQN = t.fullyQualifiedName WHERE tu.targetFQN = :targetFQN ORDER BY tu.tagFQN") + "SELECT tu.source, tu.tagFQN, tu.labelType, tu.state, t.json ->> '$.description' " + + "AS description FROM tag_usage tu " + + "LEFT JOIN tag t ON tu.tagFQN = t.fullyQualifiedName WHERE tu.targetFQN = :targetFQN ORDER BY tu.tagFQN") List getTags(@Bind("targetFQN") String targetFQN); - @SqlQuery("SELECT COUNT(*) FROM tag_usage WHERE tagFQN LIKE CONCAT(:fqnPrefix, '%')") - int getTagCount(@Bind("fqnPrefix") String fqnPrefix); + @SqlQuery("SELECT COUNT(*) FROM tag_usage " + "WHERE tagFQN LIKE CONCAT(:fqnPrefix, '%') AND source = :source") + int getTagCount(@Bind("source") int source, @Bind("fqnPrefix") String fqnPrefix); @SqlUpdate("DELETE FROM tag_usage where targetFQN = :targetFQN") void deleteTags(@Bind("targetFQN") String targetFQN); @@ -1076,6 +1078,7 @@ public interface CollectionDAO { @Override public TagLabel map(ResultSet r, StatementContext ctx) throws SQLException { return new TagLabel() + .withSource(TagLabel.Source.values()[r.getInt("source")]) .withLabelType(TagLabel.LabelType.values()[r.getInt("labelType")]) .withState(TagLabel.State.values()[r.getInt("state")]) .withTagFQN(r.getString("tagFQN")) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java index 3ec5e317768..eef1beb64f3 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java @@ -134,7 +134,7 @@ public class DashboardRepository extends EntityRepository { populateService(dashboard); dashboard.setFullyQualifiedName(getFQN(dashboard)); EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), dashboard.getOwner()); // Validate owner - dashboard.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), dashboard.getTags())); + dashboard.setTags(addDerivedTags(dashboard.getTags())); dashboard.setCharts(getCharts(dashboard.getCharts())); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java index 74f5a7a1a82..2c0f27f4801 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java @@ -17,6 +17,7 @@ import static org.openmetadata.catalog.Entity.FIELD_DESCRIPTION; import static org.openmetadata.catalog.Entity.FIELD_OWNER; import static org.openmetadata.catalog.Entity.helper; import static org.openmetadata.catalog.type.Include.DELETED; +import static org.openmetadata.catalog.util.EntityUtil.compareTagLabel; import static org.openmetadata.catalog.util.EntityUtil.entityReferenceMatch; import static org.openmetadata.catalog.util.EntityUtil.nextMajorVersion; import static org.openmetadata.catalog.util.EntityUtil.nextVersion; @@ -65,7 +66,10 @@ import org.openmetadata.catalog.type.EventType; import org.openmetadata.catalog.type.FieldChange; import org.openmetadata.catalog.type.Include; import org.openmetadata.catalog.type.Relationship; +import org.openmetadata.catalog.type.Tag; import org.openmetadata.catalog.type.TagLabel; +import org.openmetadata.catalog.type.TagLabel.LabelType; +import org.openmetadata.catalog.type.TagLabel.Source; import org.openmetadata.catalog.util.EntityInterface; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; @@ -593,16 +597,92 @@ public abstract class EntityRepository { } } + /** Validate given list of tags and add derived tags to it */ + public final List addDerivedTags(List tagLabels) throws IOException { + if (tagLabels == null || tagLabels.isEmpty()) { + return tagLabels; + } + + List updatedTagLabels = new ArrayList<>(tagLabels); + for (TagLabel tagLabel : tagLabels) { + if (tagLabel.getSource() != Source.TAG) { + // Related tags are not supported for Glossary yet + continue; + } + String json = daoCollection.tagDAO().findTag(tagLabel.getTagFQN()); + if (json == null) { + // Invalid TagLabel + throw EntityNotFoundException.byMessage( + CatalogExceptionMessage.entityNotFound(Tag.class.getSimpleName(), tagLabel.getTagFQN())); + } + Tag tag = JsonUtils.readValue(json, Tag.class); + + // Apply derived tags + List derivedTags = getDerivedTags(tagLabel, tag); + EntityUtil.mergeTags(updatedTagLabels, derivedTags); + } + updatedTagLabels.sort(compareTagLabel); + return updatedTagLabels; + } + + /** Get tags associated with a given set of tags */ + private List getDerivedTags(TagLabel tagLabel, Tag tag) throws IOException { + List derivedTags = new ArrayList<>(); + for (String fqn : listOrEmpty(tag.getAssociatedTags())) { + String json = daoCollection.tagDAO().findTag(fqn); + if (json == null) { + // Invalid TagLabel + throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Tag.class.getSimpleName(), fqn)); + } + Tag tempTag = JsonUtils.readValue(json, Tag.class); + derivedTags.add( + new TagLabel() + .withTagFQN(fqn) + .withState(tagLabel.getState()) + .withDescription(tempTag.getDescription()) + .withLabelType(LabelType.DERIVED)); + } + return derivedTags; + } + protected void applyTags(T entity) { if (supportsTags) { // Add entity level tags by adding tag to the entity relationship EntityInterface entityInterface = getEntityInterface(entity); - EntityUtil.applyTags(daoCollection.tagDAO(), entityInterface.getTags(), entityInterface.getFullyQualifiedName()); + applyTags(entityInterface.getTags(), entityInterface.getFullyQualifiedName()); // Update tag to handle additional derived tags entityInterface.setTags(getTags(entityInterface.getFullyQualifiedName())); } } + /** Apply tags {@code tagLabels} to the entity or field identified by {@code targetFQN} */ + public void applyTags(List tagLabels, String targetFQN) { + for (TagLabel tagLabel : listOrEmpty(tagLabels)) { + String json = null; + if (tagLabel.getSource() == Source.TAG) { + json = daoCollection.tagDAO().findTag(tagLabel.getTagFQN()); + } else if (tagLabel.getSource() == Source.GLOSSARY) { + json = daoCollection.glossaryTermDAO().findJsonByFqn(tagLabel.getTagFQN(), Include.NON_DELETED); + } + + if (json == null) { + // Invalid TagLabel + throw EntityNotFoundException.byMessage( + CatalogExceptionMessage.entityNotFound(Tag.class.getSimpleName(), tagLabel.getTagFQN())); + } + + // Apply tagLabel to targetFQN that identifies an entity or field + daoCollection + .tagDAO() + .applyTag( + tagLabel.getSource().ordinal(), + tagLabel.getTagFQN(), + targetFQN, + tagLabel.getLabelType().ordinal(), + tagLabel.getState().ordinal()); + } + } + protected List getTags(String fqn) { return !supportsTags ? null : daoCollection.tagDAO().getTags(fqn); } @@ -889,20 +969,16 @@ public abstract class EntityRepository { public List findFrom( UUID toId, String toEntityType, Relationship relationship, String fromEntityType, Boolean deleted) { - List ret = - daoCollection - .relationshipDAO() - .findFrom(toId.toString(), toEntityType, relationship.ordinal(), fromEntityType, deleted); - return ret; + return daoCollection + .relationshipDAO() + .findFrom(toId.toString(), toEntityType, relationship.ordinal(), fromEntityType, deleted); } public List findTo( UUID fromId, String fromEntityType, Relationship relationship, String toEntityType, Boolean deleted) { - List ret = - daoCollection - .relationshipDAO() - .findTo(fromId.toString(), fromEntityType, relationship.ordinal(), toEntityType, deleted); - return ret; + return daoCollection + .relationshipDAO() + .findTo(fromId.toString(), fromEntityType, relationship.ordinal(), toEntityType, deleted); } public void deleteTo(UUID toId, String toEntityType, Relationship relationship, String fromEntityType) { @@ -1066,7 +1142,8 @@ public abstract class EntityRepository { } // Remove current entity tags in the database. It will be added back later from the merged tag list. - EntityUtil.removeTagsByPrefix(daoCollection.tagDAO(), fqn); + daoCollection.tagDAO().deleteTagsByPrefix(fqn); + if (operation.isPut()) { // PUT operation merges tags in the request with what already exists EntityUtil.mergeTags(updatedTags, origTags); @@ -1075,8 +1152,8 @@ public abstract class EntityRepository { List addedTags = new ArrayList<>(); List deletedTags = new ArrayList<>(); recordListChange(fieldName, origTags, updatedTags, addedTags, deletedTags, EntityUtil.tagLabelMatch); - updatedTags.sort(EntityUtil.compareTagLabel); - EntityUtil.applyTags(daoCollection.tagDAO(), updatedTags, fqn); + updatedTags.sort(compareTagLabel); + applyTags(updatedTags, fqn); } public final boolean updateVersion(Double oldVersion) { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java index a04a7923280..b1641a34fa0 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java @@ -34,6 +34,7 @@ import org.openmetadata.catalog.type.ChangeDescription; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.Relationship; import org.openmetadata.catalog.type.TagLabel; +import org.openmetadata.catalog.type.TagLabel.Source; import org.openmetadata.catalog.util.EntityInterface; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; @@ -67,14 +68,14 @@ public class GlossaryRepository extends EntityRepository { glossary.setOwner(fields.contains(FIELD_OWNER) ? getOwner(glossary) : null); glossary.setTags(fields.contains("tags") ? getTags(glossary.getName()) : null); glossary.setReviewers(fields.contains("reviewers") ? getReviewers(glossary) : null); - return glossary; + return glossary.withUsageCount(fields.contains("usageCount") ? getUsageCount(glossary) : null); } @Override public void prepare(Glossary glossary) throws IOException, ParseException { glossary.setOwner(helper(glossary).validateOwnerOrNull()); validateUsers(glossary.getReviewers()); - glossary.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), glossary.getTags())); + glossary.setTags(addDerivedTags(glossary.getTags())); } @Override @@ -106,6 +107,10 @@ public class GlossaryRepository extends EntityRepository { } } + private Integer getUsageCount(Glossary glossary) { + return daoCollection.tagDAO().getTagCount(Source.GLOSSARY.ordinal(), glossary.getName()); + } + @Override public EntityInterface getEntityInterface(Glossary entity) { return new GlossaryEntityInterface(entity); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java index 176907f1c0e..1dbeb39e9d7 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java @@ -36,6 +36,7 @@ import org.openmetadata.catalog.type.ChangeDescription; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.Relationship; import org.openmetadata.catalog.type.TagLabel; +import org.openmetadata.catalog.type.TagLabel.Source; import org.openmetadata.catalog.util.EntityInterface; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; @@ -70,9 +71,14 @@ public class GlossaryTermRepository extends EntityRepository { entity.setRelatedTerms(fields.contains("relatedTerms") ? getRelatedTerms(entity) : null); entity.setReviewers(fields.contains("reviewers") ? getReviewers(entity) : null); entity.setTags(fields.contains("tags") ? getTags(entity.getFullyQualifiedName()) : null); + entity.setUsageCount(fields.contains("usageCount") ? getUsageCount(entity) : null); return entity; } + private Integer getUsageCount(GlossaryTerm term) { + return daoCollection.tagDAO().getTagCount(Source.GLOSSARY.ordinal(), term.getFullyQualifiedName()); + } + private EntityReference getParent(GlossaryTerm entity) throws IOException { List ids = findFrom( @@ -121,7 +127,7 @@ public class GlossaryTermRepository extends EntityRepository { EntityUtil.populateEntityReferences(entity.getReviewers()); // Set tags - entity.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), entity.getTags())); + entity.setTags(addDerivedTags(entity.getTags())); } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java index dc713bf8724..f226675fcdb 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java @@ -185,7 +185,7 @@ public class LocationRepository extends EntityRepository { location.setServiceType(storageService.getServiceType()); location.setFullyQualifiedName(getFQN(location)); EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), location.getOwner()); // Validate owner - location.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), location.getTags())); + location.setTags(addDerivedTags(location.getTags())); } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MetricsRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MetricsRepository.java index 2a57d3f6a22..d89496e4205 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MetricsRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MetricsRepository.java @@ -75,7 +75,7 @@ public class MetricsRepository extends EntityRepository { metrics.setFullyQualifiedName(getFQN(metrics)); EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), metrics.getOwner()); // Validate owner metrics.setService(getService(metrics.getService())); - metrics.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), metrics.getTags())); + metrics.setTags(addDerivedTags(metrics.getTags())); } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java index e67a0851235..e9a387d45a3 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java @@ -152,7 +152,7 @@ public class MlModelRepository extends EntityRepository { daoCollection.dashboardDAO().findEntityReferenceById(mlModel.getDashboard().getId()); } - mlModel.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), mlModel.getTags())); + mlModel.setTags(addDerivedTags(mlModel.getTags())); } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java index 5b7613bc025..e604093d488 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java @@ -168,7 +168,7 @@ public class PipelineRepository extends EntityRepository { populateService(pipeline); pipeline.setFullyQualifiedName(getFQN(pipeline)); EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), pipeline.getOwner()); // Validate owner - pipeline.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), pipeline.getTags())); + pipeline.setTags(addDerivedTags(pipeline.getTags())); } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java index 08ee7cf1049..5b98501d4e0 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java @@ -443,15 +443,15 @@ public class TableRepository extends EntityRepository { }); } - private void addDerivedTags(List columns) throws IOException { + private void addDerivedColumnTags(List columns) throws IOException { if (columns == null || columns.isEmpty()) { return; } for (Column column : columns) { - column.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), column.getTags())); + column.setTags(addDerivedTags(column.getTags())); if (column.getChildren() != null) { - addDerivedTags(column.getChildren()); + addDerivedColumnTags(column.getChildren()); } } } @@ -475,10 +475,10 @@ public class TableRepository extends EntityRepository
{ table.setOwner(helper(table).validateOwnerOrNull()); // Validate table tags and add derived tags to the list - table.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), table.getTags())); + table.setTags(addDerivedTags(table.getTags())); // Validate column tags - addDerivedTags(table.getColumns()); + addDerivedColumnTags(table.getColumns()); } private EntityReference getDatabase(Table table) throws IOException, ParseException { @@ -564,7 +564,7 @@ public class TableRepository extends EntityRepository
{ private void applyTags(List columns) { // Add column level tags by adding tag to column relationship for (Column column : columns) { - EntityUtil.applyTags(daoCollection.tagDAO(), column.getTags(), column.getFullyQualifiedName()); + applyTags(column.getTags(), column.getFullyQualifiedName()); if (column.getChildren() != null) { applyTags(column.getChildren()); } @@ -1023,11 +1023,11 @@ public class TableRepository extends EntityRepository
{ } // Delete tags related to deleted columns - deletedColumns.forEach(deleted -> EntityUtil.removeTags(daoCollection.tagDAO(), deleted.getFullyQualifiedName())); + deletedColumns.forEach(deleted -> daoCollection.tagDAO().deleteTags(deleted.getFullyQualifiedName())); // Add tags related to newly added columns for (Column added : addedColumns) { - EntityUtil.applyTags(daoCollection.tagDAO(), added.getTags(), added.getFullyQualifiedName()); + applyTags(added.getTags(), added.getFullyQualifiedName()); } // Carry forward the user generated metadata from existing columns to new columns diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TagDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TagDAO.java deleted file mode 100644 index bdd756dffc2..00000000000 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TagDAO.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2021 Collate - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.openmetadata.catalog.jdbi3; - -import java.util.List; -import org.jdbi.v3.sqlobject.config.RegisterRowMapper; -import org.jdbi.v3.sqlobject.customizer.Bind; -import org.jdbi.v3.sqlobject.statement.SqlQuery; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; -import org.openmetadata.catalog.jdbi3.TagRepository.TagLabelMapper; -import org.openmetadata.catalog.type.TagLabel; - -/** - * Tag categories are stored as JSON in {@code tag_category} table. All the attributes are stored as JSON document - * except href, usageCount and children tags which are constructed on the fly as needed. - * - *

Tags are stored as JSON in {@code tag} table. All the attributes of tags are stored as JSON document except href, - * usageCount and children tags which are constructed on the fly as needed. - */ -@RegisterRowMapper(TagLabelMapper.class) -public interface TagDAO { - @SqlUpdate("INSERT INTO tag_category (json) VALUES (:json)") - void insertCategory(@Bind("json") String json); - - @SqlUpdate("INSERT INTO tag(json) VALUES (:json)") - void insertTag(@Bind("json") String json); - - @SqlUpdate("UPDATE tag_category SET json = :json where name = :name") - void updateCategory(@Bind("name") String name, @Bind("json") String json); - - @SqlUpdate("UPDATE tag SET json = :json where fullyQualifiedName = :fqn") - void updateTag(@Bind("fqn") String fqn, @Bind("json") String json); - - @SqlQuery("SELECT json FROM tag_category ORDER BY name") - List listCategories(); - - @SqlQuery("SELECT json FROM tag WHERE fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') ORDER BY fullyQualifiedName") - List listChildrenTags(@Bind("fqnPrefix") String fqnPrefix); - - @SqlQuery("SELECT json FROM tag_category WHERE name = :name") - String findCategory(@Bind("name") String name); - - @SqlQuery("SELECT EXISTS (SELECT * FROM tag WHERE fullyQualifiedName = :fqn)") - boolean tagExists(@Bind("fqn") String fqn); - - @SqlQuery("SELECT json FROM tag WHERE fullyQualifiedName = :fqn") - String findTag(@Bind("fqn") String fqn); - - @SqlUpdate( - "INSERT IGNORE INTO tag_usage (tagFQN, targetFQN, labelType, state) VALUES (:tagFQN, :targetFQN, " - + ":labelType, :state)") - void applyTag( - @Bind("tagFQN") String tagFQN, - @Bind("targetFQN") String targetFQN, - @Bind("labelType") int labelType, - @Bind("state") int state); - - @SqlQuery("SELECT tagFQN, labelType, state FROM tag_usage WHERE targetFQN = :targetFQN ORDER BY tagFQN") - List getTags(@Bind("targetFQN") String targetFQN); - - @SqlQuery("SELECT COUNT(*) FROM tag_usage WHERE tagFQN LIKE CONCAT(:fqnPrefix, '%')") - int getTagCount(@Bind("fqnPrefix") String fqnPrefix); - - @SqlUpdate("DELETE FROM tag_usage where targetFQN = :targetFQN") - void deleteTags(@Bind("targetFQN") String targetFQN); - - @SqlUpdate("DELETE FROM tag_usage where targetFQN LIKE CONCAT(:fqnPrefix, '%')") - void deleteTagsByPrefix(@Bind("fqnPrefix") String fqnPrefix); -} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TagRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TagRepository.java index 1f83edfe029..40f0c223471 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TagRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TagRepository.java @@ -27,6 +27,7 @@ import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.catalog.type.Tag; import org.openmetadata.catalog.type.TagCategory; import org.openmetadata.catalog.type.TagLabel; +import org.openmetadata.catalog.type.TagLabel.Source; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.JsonUtils; @@ -251,11 +252,11 @@ public class TagRepository { } private Integer getUsageCount(TagCategory category) { - return dao.tagDAO().getTagCount(category.getName()); + return dao.tagDAO().getTagCount(Source.TAG.ordinal(), category.getName()); } private Integer getUsageCount(Tag tag) { - return dao.tagDAO().getTagCount(tag.getFullyQualifiedName()); + return dao.tagDAO().getTagCount(Source.TAG.ordinal(), tag.getFullyQualifiedName()); } public static class TagLabelMapper implements RowMapper { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java index 753659a1edd..53ff7f72f26 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java @@ -75,7 +75,7 @@ public class TopicRepository extends EntityRepository { topic.setServiceType(messagingService.getServiceType()); topic.setFullyQualifiedName(getFQN(topic)); topic.setOwner(helper(topic).validateOwnerOrNull()); - topic.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), topic.getTags())); + topic.setTags(addDerivedTags(topic.getTags())); } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java index 2d9fa0b08cc..b786afc2bd1 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java @@ -111,7 +111,7 @@ public class GlossaryResource { } } - static final String FIELDS = "owner,tags,reviewers"; + static final String FIELDS = "owner,tags,reviewers,usageCount"; public static final List ALLOWED_FIELDS = Entity.getEntityFields(Glossary.class); @GET diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResource.java index ccca7ca6451..12398e2755c 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResource.java @@ -116,7 +116,7 @@ public class GlossaryTermResource { } } - static final String FIELDS = "children,relatedTerms,reviewers,tags"; + static final String FIELDS = "children,relatedTerms,reviewers,tags,usageCount"; public static final List ALLOWED_FIELDS = Entity.getEntityFields(GlossaryTerm.class); @GET diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java index 400b5a09f1f..a1fd086e34e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java @@ -15,7 +15,6 @@ package org.openmetadata.catalog.util; import static org.openmetadata.catalog.type.Include.ALL; import static org.openmetadata.catalog.type.Include.DELETED; -import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import java.io.IOException; import java.util.ArrayList; @@ -44,7 +43,6 @@ import org.openmetadata.catalog.exception.CatalogExceptionMessage; import org.openmetadata.catalog.exception.EntityNotFoundException; import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityRelationshipDAO; import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityVersionPair; -import org.openmetadata.catalog.jdbi3.CollectionDAO.TagDAO; import org.openmetadata.catalog.jdbi3.CollectionDAO.TeamDAO; import org.openmetadata.catalog.jdbi3.CollectionDAO.UsageDAO; import org.openmetadata.catalog.jdbi3.CollectionDAO.UserDAO; @@ -62,9 +60,7 @@ import org.openmetadata.catalog.type.MlHyperParameter; import org.openmetadata.catalog.type.Relationship; import org.openmetadata.catalog.type.Schedule; import org.openmetadata.catalog.type.TableConstraint; -import org.openmetadata.catalog.type.Tag; import org.openmetadata.catalog.type.TagLabel; -import org.openmetadata.catalog.type.TagLabel.LabelType; import org.openmetadata.catalog.type.Task; import org.openmetadata.catalog.type.UsageDetails; import org.openmetadata.catalog.type.UsageStats; @@ -279,73 +275,6 @@ public final class EntityUtil { return details; } - /** Apply tags {@code tagLabels} to the entity or field identified by {@code targetFQN} */ - public static void applyTags(TagDAO tagDAO, List tagLabels, String targetFQN) { - for (TagLabel tagLabel : listOrEmpty(tagLabels)) { - String json = tagDAO.findTag(tagLabel.getTagFQN()); - if (json == null) { - // Invalid TagLabel - throw EntityNotFoundException.byMessage( - CatalogExceptionMessage.entityNotFound(Tag.class.getSimpleName(), tagLabel.getTagFQN())); - } - - // Apply tagLabel to targetFQN that identifies an entity or field - tagDAO.applyTag( - tagLabel.getTagFQN(), targetFQN, tagLabel.getLabelType().ordinal(), tagLabel.getState().ordinal()); - } - } - - public static List getDerivedTags(TagDAO tagDAO, TagLabel tagLabel, Tag tag) throws IOException { - List derivedTags = new ArrayList<>(); - for (String fqn : listOrEmpty(tag.getAssociatedTags())) { - String json = tagDAO.findTag(fqn); - if (json == null) { - // Invalid TagLabel - throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Tag.class.getSimpleName(), fqn)); - } - Tag tempTag = JsonUtils.readValue(json, Tag.class); - derivedTags.add( - new TagLabel() - .withTagFQN(fqn) - .withState(tagLabel.getState()) - .withDescription(tempTag.getDescription()) - .withLabelType(LabelType.DERIVED)); - } - return derivedTags; - } - - /** Validate given list of tags and add derived tags to it */ - public static List addDerivedTags(TagDAO tagDAO, List tagLabels) throws IOException { - if (tagLabels == null || tagLabels.isEmpty()) { - return tagLabels; - } - - List updatedTagLabels = new ArrayList<>(tagLabels); - for (TagLabel tagLabel : tagLabels) { - String json = tagDAO.findTag(tagLabel.getTagFQN()); - if (json == null) { - // Invalid TagLabel - throw EntityNotFoundException.byMessage( - CatalogExceptionMessage.entityNotFound(Tag.class.getSimpleName(), tagLabel.getTagFQN())); - } - Tag tag = JsonUtils.readValue(json, Tag.class); - - // Apply derived tags - List derivedTags = getDerivedTags(tagDAO, tagLabel, tag); - mergeTags(updatedTagLabels, derivedTags); - } - updatedTagLabels.sort(compareTagLabel); - return updatedTagLabels; - } - - public static void removeTags(TagDAO tagDAO, String fullyQualifiedName) { - tagDAO.deleteTags(fullyQualifiedName); - } - - public static void removeTagsByPrefix(TagDAO tagDAO, String fullyQualifiedName) { - tagDAO.deleteTagsByPrefix(fullyQualifiedName); - } - /** Merge derivedTags into tags, if it already does not exist in tags */ public static void mergeTags(List tags, List derivedTags) { if (derivedTags == null || derivedTags.isEmpty()) { diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/glossary.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/glossary.json index b14a3386c27..83307f26a08 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/data/glossary.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/glossary.json @@ -57,6 +57,10 @@ "description": "Owner of this glossary.", "$ref": "../../type/entityReference.json" }, + "usageCount": { + "description": "Count of how many times terms from this glossary are used.", + "type": "integer" + }, "tags": { "description": "Tags for this glossary.", "type": "array", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/glossaryTerm.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/glossaryTerm.json index 7134bc28080..71d23d68c3f 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/data/glossaryTerm.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/glossaryTerm.json @@ -105,6 +105,10 @@ "description": "User names of the reviewers for this glossary.", "$ref": "../../type/entityReference.json#/definitions/entityReferenceList" }, + "usageCount": { + "description": "Count of how many times this and it's children glossary terms are used as labels.", + "type": "integer" + }, "tags": { "description": "Tags for this glossary term.", "type": "array", diff --git a/catalog-rest-service/src/main/resources/json/schema/type/tagLabel.json b/catalog-rest-service/src/main/resources/json/schema/type/tagLabel.json index 50fb4e458d1..438254b4b9d 100644 --- a/catalog-rest-service/src/main/resources/json/schema/type/tagLabel.json +++ b/catalog-rest-service/src/main/resources/json/schema/type/tagLabel.json @@ -19,6 +19,12 @@ "description": "Unique name of the tag category.", "type": "string" }, + "source": { + "description": "Label is from Tags or Glossary", + "type": "string", + "enum": ["Tag", "Glossary"], + "default": "Tag" + }, "labelType": { "description": "Label type describes how a tag label was applied. 'Manual' indicates the tag label was applied by a person. 'Derived' indicates a tag label was derived using the associated tag relationship (see TagCategory.json for more details). 'Propagated` indicates a tag label was propagated from upstream based on lineage. 'Automated' is used when a tool was used to determine the tag label.", "type": "string", @@ -36,6 +42,6 @@ "$ref": "basic.json#/definitions/href" } }, - "required": ["tagFQN", "labelType", "state"], + "required": ["tagFQN", "source", "labelType", "state"], "additionalProperties": false } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/EnumBackwardCompatibilityTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/EnumBackwardCompatibilityTest.java index c9ca093a91c..347d9a2fc09 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/EnumBackwardCompatibilityTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/EnumBackwardCompatibilityTest.java @@ -19,16 +19,18 @@ import org.junit.jupiter.api.Test; import org.openmetadata.catalog.type.Relationship; import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.type.TagLabel.LabelType; +import org.openmetadata.catalog.type.TagLabel.Source; import org.openmetadata.catalog.type.TagLabel.State; /** * Enum ordinal number is stored in the database. New enums must be added at the end to ensure backward compatibility + * + *

Any time a new enum is added in the middle instead of at the end or enum ordinal value change, this test will + * fail. Update the test with total number of enums and test the ordinal number of the last enum. This will help catch + * new enum inadvertently being added in the middle. */ class EnumBackwardCompatibilityTest { - /** - * Any time a new enum is added, this test will fail. Update the test with total number of enums and test the ordinal - * number of the last enum. This will help catch new enum inadvertently being added in the middle. - */ + /** */ @Test void testRelationshipEnumBackwardCompatible() { assertEquals(17, Relationship.values().length); @@ -54,4 +56,14 @@ class EnumBackwardCompatibilityTest { assertEquals(2, TagLabel.State.values().length); assertEquals(1, State.CONFIRMED.ordinal()); } + + /** + * Any time a new enum is added, this test will fail. Update the test with total number of enums and test the ordinal + * number of the last enum. This will help catch new enum inadvertently being added in the middle. + */ + @Test + void testTagSourceEnumBackwardCompatible() { + assertEquals(0, Source.TAG.ordinal()); + assertEquals(1, Source.GLOSSARY.ordinal()); + } } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java index 36d71ea1154..3cfe73dbb60 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java @@ -80,6 +80,8 @@ import org.openmetadata.catalog.CatalogApplicationTest; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.api.data.CreateChart; import org.openmetadata.catalog.api.data.CreateDatabase; +import org.openmetadata.catalog.api.data.CreateGlossary; +import org.openmetadata.catalog.api.data.CreateGlossaryTerm; import org.openmetadata.catalog.api.data.TermReference; import org.openmetadata.catalog.api.services.CreateDashboardService; import org.openmetadata.catalog.api.services.CreateDashboardService.DashboardServiceType; @@ -92,6 +94,8 @@ import org.openmetadata.catalog.api.services.CreatePipelineService.PipelineServi import org.openmetadata.catalog.api.services.CreateStorageService; import org.openmetadata.catalog.entity.data.Chart; import org.openmetadata.catalog.entity.data.Database; +import org.openmetadata.catalog.entity.data.Glossary; +import org.openmetadata.catalog.entity.data.GlossaryTerm; import org.openmetadata.catalog.entity.services.DashboardService; import org.openmetadata.catalog.entity.services.DatabaseService; import org.openmetadata.catalog.entity.services.MessagingService; @@ -104,6 +108,8 @@ import org.openmetadata.catalog.jdbi3.ChartRepository.ChartEntityInterface; import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface; import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface; import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface; +import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface; +import org.openmetadata.catalog.jdbi3.GlossaryTermRepository.GlossaryTermEntityInterface; import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface; import org.openmetadata.catalog.jdbi3.PipelineServiceRepository.PipelineServiceEntityInterface; import org.openmetadata.catalog.jdbi3.RoleRepository.RoleEntityInterface; @@ -114,6 +120,8 @@ import org.openmetadata.catalog.resources.charts.ChartResourceTest; import org.openmetadata.catalog.resources.databases.DatabaseResourceTest; import org.openmetadata.catalog.resources.events.EventResource.ChangeEventList; import org.openmetadata.catalog.resources.events.WebhookResourceTest; +import org.openmetadata.catalog.resources.glossary.GlossaryResourceTest; +import org.openmetadata.catalog.resources.glossary.GlossaryTermResourceTest; import org.openmetadata.catalog.resources.services.DashboardServiceResourceTest; import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest; import org.openmetadata.catalog.resources.services.MessagingServiceResourceTest; @@ -136,6 +144,7 @@ import org.openmetadata.catalog.type.Include; import org.openmetadata.catalog.type.StorageServiceType; import org.openmetadata.catalog.type.Tag; import org.openmetadata.catalog.type.TagLabel; +import org.openmetadata.catalog.type.TagLabel.Source; import org.openmetadata.catalog.util.EntityInterface; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.JsonUtils; @@ -192,12 +201,24 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { public static EntityReference GCP_STORAGE_SERVICE_REFERENCE; public static TagLabel USER_ADDRESS_TAG_LABEL; - public static TagLabel USER_BANK_ACCOUNT_TAG_LABEL; public static TagLabel PERSONAL_DATA_TAG_LABEL; public static TagLabel PII_SENSITIVE_TAG_LABEL; public static TagLabel TIER1_TAG_LABEL; public static TagLabel TIER2_TAG_LABEL; + public static Glossary GLOSSARY1; + public static EntityReference GLOSSARY1_REF; + public static Glossary GLOSSARY2; + public static EntityReference GLOSSARY2_REF; + + public static GlossaryTerm GLOSSARY1_TERM1; + public static EntityReference GLOSSARY1_TERM1_REF; + public static TagLabel GLOSSARY1_TERM1_LABEL; + + public static GlossaryTerm GLOSSARY2_TERM1; + public static EntityReference GLOSSARY2_TERM1_REF; + public static TagLabel GLOSSARY2_TERM1_LABEL; + public static EntityReference SUPERSET_REFERENCE; public static EntityReference LOOKER_REFERENCE; public static List CHART_REFERENCES; @@ -353,7 +374,6 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { GCP_STORAGE_SERVICE_REFERENCE = new StorageServiceEntityInterface(service).getEntityReference(); USER_ADDRESS_TAG_LABEL = getTagLabel("User.Address"); - USER_BANK_ACCOUNT_TAG_LABEL = getTagLabel("User.BankAccount"); PERSONAL_DATA_TAG_LABEL = getTagLabel("PersonalData.Personal"); PII_SENSITIVE_TAG_LABEL = getTagLabel("PII.Sensitive"); TIER1_TAG_LABEL = getTagLabel("Tier.Tier1"); @@ -386,11 +406,39 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { DATABASE = databaseResourceTest.createAndCheckEntity(create, ADMIN_AUTH_HEADERS); DATABASE_REFERENCE = new DatabaseEntityInterface(DATABASE).getEntityReference(); + GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest(); + CreateGlossary createGlossary = glossaryResourceTest.createRequest("g1", "", "", null); + GLOSSARY1 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS); + GLOSSARY1_REF = new GlossaryEntityInterface(GLOSSARY1).getEntityReference(); + + createGlossary = glossaryResourceTest.createRequest("g2", "", "", null); + GLOSSARY2 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS); + GLOSSARY2_REF = new GlossaryEntityInterface(GLOSSARY2).getEntityReference(); + + GlossaryTermResourceTest glossaryTermResourceTest = new GlossaryTermResourceTest(); + CreateGlossaryTerm createGlossaryTerm = + glossaryTermResourceTest + .createRequest("g1t1", null, "", null) + .withRelatedTerms(null) + .withGlossary(GLOSSARY1_REF); + GLOSSARY1_TERM1 = glossaryTermResourceTest.createEntity(createGlossaryTerm, ADMIN_AUTH_HEADERS); + GLOSSARY1_TERM1_REF = new GlossaryTermEntityInterface(GLOSSARY1_TERM1).getEntityReference(); + GLOSSARY1_TERM1_LABEL = getTagLabel(GLOSSARY1_TERM1); + + createGlossaryTerm = + glossaryTermResourceTest + .createRequest("g2t1", null, "", null) + .withRelatedTerms(null) + .withGlossary(GLOSSARY2_REF); + GLOSSARY2_TERM1 = glossaryTermResourceTest.createEntity(createGlossaryTerm, ADMIN_AUTH_HEADERS); + GLOSSARY2_TERM1_REF = new GlossaryTermEntityInterface(GLOSSARY2_TERM1).getEntityReference(); + GLOSSARY2_TERM1_LABEL = getTagLabel(GLOSSARY2_TERM1); + COLUMNS = Arrays.asList( getColumn("c1", BIGINT, USER_ADDRESS_TAG_LABEL), getColumn("c2", ColumnDataType.VARCHAR, USER_ADDRESS_TAG_LABEL).withDataLength(10), - getColumn("c3", BIGINT, USER_BANK_ACCOUNT_TAG_LABEL)); + getColumn("c3", BIGINT, GLOSSARY1_TERM1_LABEL)); } private TagLabel getTagLabel(String tagName) throws HttpResponseException { @@ -398,6 +446,13 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { return new TagLabel().withTagFQN(tag.getFullyQualifiedName()).withDescription(tag.getDescription()); } + private TagLabel getTagLabel(GlossaryTerm term) { + return new TagLabel() + .withTagFQN(term.getFullyQualifiedName()) + .withDescription(term.getDescription()) + .withSource(Source.GLOSSARY); + } + @AfterAll public void afterAllTests() throws Exception { if (runWebhookTests) { @@ -1141,6 +1196,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { if (supportsTags) { entityInterface.setTags(new ArrayList<>()); entityInterface.getTags().add(USER_ADDRESS_TAG_LABEL); + entityInterface.getTags().add(GLOSSARY2_TERM1_LABEL); change.getFieldsAdded().add(new FieldChange().withName("tags").withNewValue(entityInterface.getTags())); } if (supportsDots) { diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java index 00adeb4a7af..714f31d1659 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java @@ -83,6 +83,8 @@ import org.openmetadata.catalog.entity.services.DatabaseService; import org.openmetadata.catalog.jdbi3.TableRepository.TableEntityInterface; import org.openmetadata.catalog.resources.EntityResourceTest; import org.openmetadata.catalog.resources.databases.TableResource.TableList; +import org.openmetadata.catalog.resources.glossary.GlossaryResourceTest; +import org.openmetadata.catalog.resources.glossary.GlossaryTermResourceTest; import org.openmetadata.catalog.resources.locations.LocationResourceTest; import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest; import org.openmetadata.catalog.resources.tags.TagResourceTest; @@ -253,7 +255,7 @@ public class TableResourceTest extends EntityResourceTest { getColumn("column3", STRING, null) .withOrdinalPosition(3) .withDescription("column3") - .withTags(List.of(USER_ADDRESS_TAG_LABEL, USER_BANK_ACCOUNT_TAG_LABEL)); + .withTags(List.of(USER_ADDRESS_TAG_LABEL, GLOSSARY1_TERM1_LABEL)); columns = new ArrayList<>(); columns.add(updateColumn1); columns.add(updateColumn2); @@ -330,7 +332,7 @@ public class TableResourceTest extends EntityResourceTest { // Column struct>> Column c2 = - getColumn("c2", STRUCT, "struct>", USER_BANK_ACCOUNT_TAG_LABEL) + getColumn("c2", STRUCT, "struct>", GLOSSARY1_TERM1_LABEL) .withChildren(new ArrayList<>(Arrays.asList(c2_a, c2_b, c2_c))); // Test POST operation can create complex types @@ -364,10 +366,10 @@ public class TableResourceTest extends EntityResourceTest { // struct>> // to // struct<-----, b:char, c:struct, f:char> - c2_b.withTags(List.of(USER_ADDRESS_TAG_LABEL, USER_BANK_ACCOUNT_TAG_LABEL)); // Add new tag to c2.b tag + c2_b.withTags(List.of(USER_ADDRESS_TAG_LABEL, GLOSSARY1_TERM1_LABEL)); // Add new tag to c2.b tag change .getFieldsAdded() - .add(new FieldChange().withName("columns.c2.b.tags").withNewValue(List.of(USER_BANK_ACCOUNT_TAG_LABEL))); + .add(new FieldChange().withName("columns.c2.b.tags").withNewValue(List.of(GLOSSARY1_TERM1_LABEL))); Column c2_c_e = getColumn("e", INT, USER_ADDRESS_TAG_LABEL); c2_c.getChildren().add(c2_c_e); // Add c2.c.e @@ -401,13 +403,13 @@ public class TableResourceTest extends EntityResourceTest { // String tableJson = JsonUtils.pojoToJson(table1); c1 = table1.getColumns().get(0); - c1.withTags(singletonList(USER_BANK_ACCOUNT_TAG_LABEL)); // c1 tag changed + c1.withTags(singletonList(GLOSSARY1_TERM1_LABEL)); // c1 tag changed c2 = table1.getColumns().get(1); - c2.withTags(Arrays.asList(USER_ADDRESS_TAG_LABEL, USER_BANK_ACCOUNT_TAG_LABEL)); // c2 new tag added + c2.withTags(Arrays.asList(USER_ADDRESS_TAG_LABEL, GLOSSARY1_TERM1_LABEL)); // c2 new tag added c2_a = c2.getChildren().get(0); - c2_a.withTags(singletonList(USER_BANK_ACCOUNT_TAG_LABEL)); // c2.a tag changed + c2_a.withTags(singletonList(GLOSSARY1_TERM1_LABEL)); // c2.a tag changed c2_b = c2.getChildren().get(1); c2_b.withTags(new ArrayList<>()); // c2.b tag removed @@ -416,7 +418,7 @@ public class TableResourceTest extends EntityResourceTest { c2_c.withTags(new ArrayList<>()); // c2.c tag removed c2_c_d = c2_c.getChildren().get(0); - c2_c_d.setTags(singletonList(USER_BANK_ACCOUNT_TAG_LABEL)); // c2.c.d new tag added + c2_c_d.setTags(singletonList(GLOSSARY1_TERM1_LABEL)); // c2.c.d new tag added table1 = patchEntity(table1.getId(), tableJson, table1, ADMIN_AUTH_HEADERS); assertColumns(Arrays.asList(c1, c2), table1.getColumns()); } @@ -517,7 +519,7 @@ public class TableResourceTest extends EntityResourceTest { void put_updateColumns_200(TestInfo test) throws IOException { int tagCategoryUsageCount = getTagCategoryUsageCount("user", TEST_AUTH_HEADERS); int addressTagUsageCount = getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS); - int bankTagUsageCount = getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS); + int glossaryTermUsageCount = getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), TEST_AUTH_HEADERS); // // Create a table with column c1, type BIGINT, description c1 and tag USER_ADDRESS_TAB_LABEL @@ -534,25 +536,27 @@ public class TableResourceTest extends EntityResourceTest { // Ensure tag category and tag usage counts are updated assertEquals(tagCategoryUsageCount + 1, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS)); assertEquals(addressTagUsageCount + 1, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS)); - assertEquals(bankTagUsageCount, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS)); + assertEquals( + glossaryTermUsageCount, getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), TEST_AUTH_HEADERS)); // - // Update the c1 tags to USER_ADDRESS_TAB_LABEL, USER_BANK_ACCOUNT_TAG_LABEL (newly added) + // Update the c1 tags to USER_ADDRESS_TAB_LABEL, GLOSSARY1_TERM1_LABEL (newly added) // Ensure description and previous tag is carried forward during update // - tags.add(USER_BANK_ACCOUNT_TAG_LABEL); + tags.add(GLOSSARY1_TERM1_LABEL); List updatedColumns = new ArrayList<>(); updatedColumns.add(getColumn("c1", BIGINT, null).withTags(tags)); ChangeDescription change = getChangeDescription(table.getVersion()); change .getFieldsAdded() - .add(new FieldChange().withName("columns.c1.tags").withNewValue(List.of(USER_BANK_ACCOUNT_TAG_LABEL))); + .add(new FieldChange().withName("columns.c1.tags").withNewValue(List.of(GLOSSARY1_TERM1_LABEL))); table = updateAndCheckEntity(request.withColumns(updatedColumns), OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); // Ensure tag usage counts are updated - assertEquals(tagCategoryUsageCount + 2, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS)); + assertEquals(tagCategoryUsageCount + 1, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS)); assertEquals(addressTagUsageCount + 1, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS)); - assertEquals(bankTagUsageCount + 1, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS)); + assertEquals( + glossaryTermUsageCount + 1, getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), TEST_AUTH_HEADERS)); // // Add a new column c2 using PUT @@ -563,10 +567,11 @@ public class TableResourceTest extends EntityResourceTest { change.getFieldsAdded().add(new FieldChange().withName("columns").withNewValue(List.of(c2))); table = updateAndCheckEntity(request.withColumns(updatedColumns), OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); - // Ensure tag usage counts are updated - column c2 added both address and bank tags - assertEquals(tagCategoryUsageCount + 4, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS)); + // Ensure tag usage counts are updated - column c2 added both address + assertEquals(tagCategoryUsageCount + 2, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS)); assertEquals(addressTagUsageCount + 2, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS)); - assertEquals(bankTagUsageCount + 2, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS)); + assertEquals( + glossaryTermUsageCount + 2, getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), TEST_AUTH_HEADERS)); // // Change the column c2 data length from 10 to 20. Increasing the data length is considered backward compatible @@ -597,9 +602,10 @@ public class TableResourceTest extends EntityResourceTest { assertEquals(1, table.getColumns().size()); // Ensure tag usage counts are updated to reflect removal of column c2 - assertEquals(tagCategoryUsageCount + 2, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS)); + assertEquals(tagCategoryUsageCount + 1, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS)); assertEquals(addressTagUsageCount + 1, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS)); - assertEquals(bankTagUsageCount + 1, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS)); + assertEquals( + glossaryTermUsageCount + 1, getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), TEST_AUTH_HEADERS)); } @Test @@ -1213,16 +1219,25 @@ public class TableResourceTest extends EntityResourceTest { CreateTable create = createRequest(test, 1) .withOwner(USER_OWNER1) - .withTags(singletonList(USER_ADDRESS_TAG_LABEL)) // 1 table tag - USER_ADDRESS - .withColumns(COLUMNS); // 3 column tags - 2 USER_ADDRESS and 1 USER_BANK_ACCOUNT + .withTags(List.of(USER_ADDRESS_TAG_LABEL, GLOSSARY2_TERM1_LABEL)) // 2 table tags - USER_ADDRESS, g2t1 + .withColumns(COLUMNS); // 3 column tags - 2 USER_ADDRESS and 1 g1t1 createAndCheckEntity(create, ADMIN_AUTH_HEADERS); - // Total 4 user tags - 1 table 1 tag + 3 column tags - assertEquals(4, getTagCategoryUsageCount("user", ADMIN_AUTH_HEADERS)); - // Total 3 USER_ADDRESS tags - 1 table 1 tag and 2 column tags + // Total 3 user tags - 1 table tag + 2 column tags + assertEquals(3, getTagCategoryUsageCount("user", ADMIN_AUTH_HEADERS)); + + // Total 1 glossary1 tags - 1 column + assertEquals(1, getGlossaryUsageCount("g1", ADMIN_AUTH_HEADERS)); + + // Total 1 glossary2 tags - 1 table + assertEquals(1, getGlossaryUsageCount("g2", ADMIN_AUTH_HEADERS)); + + // Total 3 USER_ADDRESS tags - 1 table tag and 2 column tags assertEquals(3, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS)); - // Total 1 USER_BANK_ACCOUNT tags - 1 column level - assertEquals(1, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS)); + // Total 1 GLOSSARY1_TERM1 - 1 column level + assertEquals(1, getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS)); + // Total 1 GLOSSARY1_TERM1 - 1 table level + assertEquals(1, getGlossaryTermUsageCount(GLOSSARY2_TERM1_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS)); // Create a table test2 with 3 column tags CreateTable create1 = @@ -1232,12 +1247,12 @@ public class TableResourceTest extends EntityResourceTest { .withColumns(COLUMNS); // 3 column tags - 2 USER_ADDRESS and 1 USER_BANK_ACCOUNT createAndCheckEntity(create1, ADMIN_AUTH_HEADERS); - // Additional 3 user tags - 3 column tags - assertEquals(7, getTagCategoryUsageCount("user", ADMIN_AUTH_HEADERS)); + // Additional 2 user tags - 2 column tags + assertEquals(5, getTagCategoryUsageCount("user", ADMIN_AUTH_HEADERS)); // Additional 2 USER_ADDRESS tags - 2 column tags assertEquals(5, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS)); - // Additional 2 USER_BANK_ACCOUNT tags - 1 column tags - assertEquals(2, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS)); + // Additional 1 glossary tag - 1 column tags + assertEquals(2, getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS)); ResultList

tableList = listEntities(null, ADMIN_AUTH_HEADERS); // List tables assertEquals(2, tableList.getData().size()); @@ -1372,7 +1387,7 @@ public class TableResourceTest extends EntityResourceTest { List columns = new ArrayList<>(); columns.add(getColumn("c1", INT, USER_ADDRESS_TAG_LABEL).withDescription(null)); columns.add(getColumn("c2", BIGINT, USER_ADDRESS_TAG_LABEL)); - columns.add(getColumn("c3", FLOAT, USER_BANK_ACCOUNT_TAG_LABEL)); + columns.add(getColumn("c3", FLOAT, GLOSSARY1_TERM1_LABEL)); Table table = createEntity(createRequest(test).withColumns(columns), ADMIN_AUTH_HEADERS); @@ -1381,7 +1396,7 @@ public class TableResourceTest extends EntityResourceTest { columns .get(0) .withDescription("new0") // Set new description - .withTags(List.of(USER_ADDRESS_TAG_LABEL, USER_BANK_ACCOUNT_TAG_LABEL)); + .withTags(List.of(USER_ADDRESS_TAG_LABEL, GLOSSARY1_TERM1_LABEL)); change .getFieldsAdded() .add( @@ -1391,7 +1406,7 @@ public class TableResourceTest extends EntityResourceTest { .add( new FieldChange() .withName("columns.c1.tags") - .withNewValue(List.of(USER_BANK_ACCOUNT_TAG_LABEL))); // Column c1 got new tags + .withNewValue(List.of(GLOSSARY1_TERM1_LABEL))); // Column c1 got new tags columns .get(1) @@ -1411,7 +1426,7 @@ public class TableResourceTest extends EntityResourceTest { .add( new FieldChange() .withName("columns.c3.tags") - .withOldValue(List.of(USER_BANK_ACCOUNT_TAG_LABEL))); // Column c3 tags were removed + .withOldValue(List.of(GLOSSARY1_TERM1_LABEL))); // Column c3 tags were removed String originalJson = JsonUtils.pojoToJson(table); table.setColumns(columns); @@ -1705,6 +1720,15 @@ public class TableResourceTest extends EntityResourceTest { return TagResourceTest.getCategory(name, "usageCount", authHeaders).getUsageCount(); } + private static int getGlossaryUsageCount(String name, Map authHeaders) throws HttpResponseException { + return new GlossaryResourceTest().getEntityByName(name, "usageCount", authHeaders).getUsageCount(); + } + + private static int getGlossaryTermUsageCount(String name, Map authHeaders) + throws HttpResponseException { + return new GlossaryTermResourceTest().getEntityByName(name, "usageCount", authHeaders).getUsageCount(); + } + private void verifyTableProfileData(List actualProfiles, List expectedProfiles) { assertEquals(actualProfiles.size(), expectedProfiles.size()); Map tableProfileMap = new HashMap<>(); @@ -1718,8 +1742,7 @@ public class TableResourceTest extends EntityResourceTest { } } - private void verifyTableTest(String tableName, List actualTests, List expectedTests) - throws IOException { + private void verifyTableTest(String tableName, List actualTests, List expectedTests) { assertEquals(expectedTests.size(), actualTests.size()); Map tableTestMap = new HashMap<>(); for (TableTest test : actualTests) { @@ -1759,7 +1782,7 @@ public class TableResourceTest extends EntityResourceTest { } } - private void verifyColumnTest(Table table, Column column, List expectedTests) throws IOException { + private void verifyColumnTest(Table table, Column column, List expectedTests) { List actualTests = new ArrayList<>(); for (Column c : table.getColumns()) { if (c.getName().equals(column.getName())) { @@ -1818,7 +1841,7 @@ public class TableResourceTest extends EntityResourceTest { } } - private void verifyTestCaseResults(TestCaseResult expected, List actual) throws IOException { + private void verifyTestCaseResults(TestCaseResult expected, List actual) { Map actualResultMap = new HashMap<>(); for (Object a : actual) { TestCaseResult result = JsonUtils.convertValue(a, TestCaseResult.class); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResourceTest.java index b46bae6ab10..87753c2efe9 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResourceTest.java @@ -30,14 +30,12 @@ import static org.openmetadata.catalog.util.TestUtils.validateEntityReference; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.http.client.HttpResponseException; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; @@ -64,16 +62,6 @@ import org.openmetadata.catalog.util.TestUtils.UpdateType; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class GlossaryTermResourceTest extends EntityResourceTest { - public static Glossary GLOSSARY1; - public static EntityReference GLOSSARY_REF1; - public static Glossary GLOSSARY2; - public static EntityReference GLOSSARY_REF2; - - public static GlossaryTerm GLOSSARY_TERM1; - public static EntityReference GLOSSARY_TERM_REF1; - public static GlossaryTerm GLOSSARY_TERM2; - public static EntityReference GLOSSARY_TERM_REF2; - public GlossaryTermResourceTest() { super( Entity.GLOSSARY_TERM, @@ -88,27 +76,6 @@ public class GlossaryTermResourceTest extends EntityResourceTest list = listEntities(queryParams, ADMIN_AUTH_HEADERS); List expectedTerms = - Arrays.asList(GLOSSARY_TERM1, GLOSSARY_TERM2, term1, term11, term12, term2, term21, term22); + Arrays.asList(GLOSSARY1_TERM1, GLOSSARY2_TERM1, term1, term11, term12, term2, term21, term22); assertContains(expectedTerms, list.getData()); // List terms under glossary1 @@ -262,8 +229,8 @@ public class GlossaryTermResourceTest extends EntityResourceTest updatedExpectedList = new ArrayList<>(expectedList); for (TagLabel expected : expectedList) { + if (expected.getSource() != Source.TAG) { + continue; // TODO similar test for glossary + } Tag tag = TagResourceTest.getTag(expected.getTagFQN(), ADMIN_AUTH_HEADERS); List derived = new ArrayList<>(); for (String fqn : listOrEmpty(tag.getAssociatedTags())) {