yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-19 14:37:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #3239 Add support for using glossary terms as tag labels (#3240)

Browse Source
This commit is contained in:
Suresh Srinivas 2022-03-07 19:35:48 -08:00 committed by GitHub
parent 46a85e90a0
commit f540981de1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 302 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
bootstrap/sql/mysql/v004__create_db_connection_info.sql
View File

@ -56,3 +56,12 @@ ALTER TABLE role_entity
ADD COLUMN `default` BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.default')),
ADD INDEX(`default`);
-- Add tag label source
ALTER TABLE tag_usage
ADD COLUMN source TINYINT NOT NULL FIRST, -- Source of tag (either from TagCategory or Glossary)
DROP KEY unique_name,
ADD UNIQUE KEY unique_name(source, tagFQN, targetFQN);
UPDATE tag_usage
SET source = 0
WHERE source IS NULL;

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java
View File

@ -67,7 +67,7 @@ public class ChartRepository extends EntityRepository<Chart> {
chart.setServiceType(dashboardService.getServiceType());
chart.setFullyQualifiedName(getFQN(chart));
chart.setOwner(helper(chart).validateOwnerOrNull());
chart.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), chart.getTags()));
chart.setTags(addDerivedTags(chart.getTags()));
}
@Override

15
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java
View File

@ -1050,21 +1050,23 @@ public interface CollectionDAO {
String findTag(@Bind("fqn") String fqn);
@SqlUpdate(
"INSERT IGNORE INTO tag_usage (tagFQN, targetFQN, labelType, state) VALUES (:tagFQN, :targetFQN, "
+ ":labelType, :state)")
"INSERT IGNORE INTO tag_usage (source, tagFQN, targetFQN, labelType, state) "
+ "VALUES (:source, :tagFQN, :targetFQN, :labelType, :state)")
void applyTag(
@Bind("source") int source,
@Bind("tagFQN") String tagFQN,
@Bind("targetFQN") String targetFQN,
@Bind("labelType") int labelType,
@Bind("state") int state);
@SqlQuery(
"SELECT tu.tagFQN, tu.labelType, tu.state, t.json ->> '$.description' AS description FROM tag_usage tu "
+ "JOIN tag t ON tu.tagFQN = t.fullyQualifiedName WHERE tu.targetFQN = :targetFQN ORDER BY tu.tagFQN")
"SELECT tu.source, tu.tagFQN, tu.labelType, tu.state, t.json ->> '$.description' "
+ "AS description FROM tag_usage tu "
+ "LEFT JOIN tag t ON tu.tagFQN = t.fullyQualifiedName WHERE tu.targetFQN = :targetFQN ORDER BY tu.tagFQN")
List<TagLabel> getTags(@Bind("targetFQN") String targetFQN);
@SqlQuery("SELECT COUNT(*) FROM tag_usage WHERE tagFQN LIKE CONCAT(:fqnPrefix, '%')")
int getTagCount(@Bind("fqnPrefix") String fqnPrefix);
@SqlQuery("SELECT COUNT(*) FROM tag_usage " + "WHERE tagFQN LIKE CONCAT(:fqnPrefix, '%') AND source = :source")
int getTagCount(@Bind("source") int source, @Bind("fqnPrefix") String fqnPrefix);
@SqlUpdate("DELETE FROM tag_usage where targetFQN = :targetFQN")
void deleteTags(@Bind("targetFQN") String targetFQN);
@ -1076,6 +1078,7 @@ public interface CollectionDAO {
@Override
public TagLabel map(ResultSet r, StatementContext ctx) throws SQLException {
return new TagLabel()
.withSource(TagLabel.Source.values()[r.getInt("source")])
.withLabelType(TagLabel.LabelType.values()[r.getInt("labelType")])
.withState(TagLabel.State.values()[r.getInt("state")])
.withTagFQN(r.getString("tagFQN"))

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java
View File

@ -134,7 +134,7 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
populateService(dashboard);
dashboard.setFullyQualifiedName(getFQN(dashboard));
EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), dashboard.getOwner()); // Validate owner
dashboard.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), dashboard.getTags()));
dashboard.setTags(addDerivedTags(dashboard.getTags()));
dashboard.setCharts(getCharts(dashboard.getCharts()));
}

105
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java
View File

@ -17,6 +17,7 @@ import static org.openmetadata.catalog.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.catalog.Entity.helper;
import static org.openmetadata.catalog.type.Include.DELETED;
import static org.openmetadata.catalog.util.EntityUtil.compareTagLabel;
import static org.openmetadata.catalog.util.EntityUtil.entityReferenceMatch;
import static org.openmetadata.catalog.util.EntityUtil.nextMajorVersion;
import static org.openmetadata.catalog.util.EntityUtil.nextVersion;
@ -65,7 +66,10 @@ import org.openmetadata.catalog.type.EventType;
import org.openmetadata.catalog.type.FieldChange;
import org.openmetadata.catalog.type.Include;
import org.openmetadata.catalog.type.Relationship;
import org.openmetadata.catalog.type.Tag;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.TagLabel.LabelType;
import org.openmetadata.catalog.type.TagLabel.Source;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields;
@ -593,16 +597,92 @@ public abstract class EntityRepository<T> {
}
}
/** Validate given list of tags and add derived tags to it */
public final List<TagLabel> addDerivedTags(List<TagLabel> tagLabels) throws IOException {
if (tagLabels == null || tagLabels.isEmpty()) {
return tagLabels;
}
List<TagLabel> updatedTagLabels = new ArrayList<>(tagLabels);
for (TagLabel tagLabel : tagLabels) {
if (tagLabel.getSource() != Source.TAG) {
// Related tags are not supported for Glossary yet
continue;
}
String json = daoCollection.tagDAO().findTag(tagLabel.getTagFQN());
if (json == null) {
// Invalid TagLabel
throw EntityNotFoundException.byMessage(
CatalogExceptionMessage.entityNotFound(Tag.class.getSimpleName(), tagLabel.getTagFQN()));
}
Tag tag = JsonUtils.readValue(json, Tag.class);
// Apply derived tags
List<TagLabel> derivedTags = getDerivedTags(tagLabel, tag);
EntityUtil.mergeTags(updatedTagLabels, derivedTags);
}
updatedTagLabels.sort(compareTagLabel);
return updatedTagLabels;
}
/** Get tags associated with a given set of tags */
private List<TagLabel> getDerivedTags(TagLabel tagLabel, Tag tag) throws IOException {
List<TagLabel> derivedTags = new ArrayList<>();
for (String fqn : listOrEmpty(tag.getAssociatedTags())) {
String json = daoCollection.tagDAO().findTag(fqn);
if (json == null) {
// Invalid TagLabel
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Tag.class.getSimpleName(), fqn));
}
Tag tempTag = JsonUtils.readValue(json, Tag.class);
derivedTags.add(
new TagLabel()
.withTagFQN(fqn)
.withState(tagLabel.getState())
.withDescription(tempTag.getDescription())
.withLabelType(LabelType.DERIVED));
}
return derivedTags;
}
protected void applyTags(T entity) {
if (supportsTags) {
// Add entity level tags by adding tag to the entity relationship
EntityInterface<T> entityInterface = getEntityInterface(entity);
EntityUtil.applyTags(daoCollection.tagDAO(), entityInterface.getTags(), entityInterface.getFullyQualifiedName());
applyTags(entityInterface.getTags(), entityInterface.getFullyQualifiedName());
// Update tag to handle additional derived tags
entityInterface.setTags(getTags(entityInterface.getFullyQualifiedName()));
}
}
/** Apply tags {@code tagLabels} to the entity or field identified by {@code targetFQN} */
public void applyTags(List<TagLabel> tagLabels, String targetFQN) {
for (TagLabel tagLabel : listOrEmpty(tagLabels)) {
String json = null;
if (tagLabel.getSource() == Source.TAG) {
json = daoCollection.tagDAO().findTag(tagLabel.getTagFQN());
} else if (tagLabel.getSource() == Source.GLOSSARY) {
json = daoCollection.glossaryTermDAO().findJsonByFqn(tagLabel.getTagFQN(), Include.NON_DELETED);
}
if (json == null) {
// Invalid TagLabel
throw EntityNotFoundException.byMessage(
CatalogExceptionMessage.entityNotFound(Tag.class.getSimpleName(), tagLabel.getTagFQN()));
}
// Apply tagLabel to targetFQN that identifies an entity or field
daoCollection
.tagDAO()
.applyTag(
tagLabel.getSource().ordinal(),
tagLabel.getTagFQN(),
targetFQN,
tagLabel.getLabelType().ordinal(),
tagLabel.getState().ordinal());
}
}
protected List<TagLabel> getTags(String fqn) {
return !supportsTags ? null : daoCollection.tagDAO().getTags(fqn);
}
@ -889,20 +969,16 @@ public abstract class EntityRepository<T> {
public List<String> findFrom(
UUID toId, String toEntityType, Relationship relationship, String fromEntityType, Boolean deleted) {
List<String> ret =
daoCollection
.relationshipDAO()
.findFrom(toId.toString(), toEntityType, relationship.ordinal(), fromEntityType, deleted);
return ret;
return daoCollection
.relationshipDAO()
.findFrom(toId.toString(), toEntityType, relationship.ordinal(), fromEntityType, deleted);
}
public List<String> findTo(
UUID fromId, String fromEntityType, Relationship relationship, String toEntityType, Boolean deleted) {
List<String> ret =
daoCollection
.relationshipDAO()
.findTo(fromId.toString(), fromEntityType, relationship.ordinal(), toEntityType, deleted);
return ret;
return daoCollection
.relationshipDAO()
.findTo(fromId.toString(), fromEntityType, relationship.ordinal(), toEntityType, deleted);
}
public void deleteTo(UUID toId, String toEntityType, Relationship relationship, String fromEntityType) {
@ -1066,7 +1142,8 @@ public abstract class EntityRepository<T> {
}
// Remove current entity tags in the database. It will be added back later from the merged tag list.
EntityUtil.removeTagsByPrefix(daoCollection.tagDAO(), fqn);
daoCollection.tagDAO().deleteTagsByPrefix(fqn);
if (operation.isPut()) {
// PUT operation merges tags in the request with what already exists
EntityUtil.mergeTags(updatedTags, origTags);
@ -1075,8 +1152,8 @@ public abstract class EntityRepository<T> {
List<TagLabel> addedTags = new ArrayList<>();
List<TagLabel> deletedTags = new ArrayList<>();
recordListChange(fieldName, origTags, updatedTags, addedTags, deletedTags, EntityUtil.tagLabelMatch);
updatedTags.sort(EntityUtil.compareTagLabel);
EntityUtil.applyTags(daoCollection.tagDAO(), updatedTags, fqn);
updatedTags.sort(compareTagLabel);
applyTags(updatedTags, fqn);
}
public final boolean updateVersion(Double oldVersion) {

9
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java
View File

@ -34,6 +34,7 @@ import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.Relationship;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.TagLabel.Source;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields;
@ -67,14 +68,14 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
glossary.setOwner(fields.contains(FIELD_OWNER) ? getOwner(glossary) : null);
glossary.setTags(fields.contains("tags") ? getTags(glossary.getName()) : null);
glossary.setReviewers(fields.contains("reviewers") ? getReviewers(glossary) : null);
return glossary;
return glossary.withUsageCount(fields.contains("usageCount") ? getUsageCount(glossary) : null);
}
@Override
public void prepare(Glossary glossary) throws IOException, ParseException {
glossary.setOwner(helper(glossary).validateOwnerOrNull());
validateUsers(glossary.getReviewers());
glossary.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), glossary.getTags()));
glossary.setTags(addDerivedTags(glossary.getTags()));
}
@Override
@ -106,6 +107,10 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
}
}
private Integer getUsageCount(Glossary glossary) {
return daoCollection.tagDAO().getTagCount(Source.GLOSSARY.ordinal(), glossary.getName());
}
@Override
public EntityInterface<Glossary> getEntityInterface(Glossary entity) {
return new GlossaryEntityInterface(entity);

8
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java
View File

@ -36,6 +36,7 @@ import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.Relationship;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.TagLabel.Source;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields;
@ -70,9 +71,14 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
entity.setRelatedTerms(fields.contains("relatedTerms") ? getRelatedTerms(entity) : null);
entity.setReviewers(fields.contains("reviewers") ? getReviewers(entity) : null);
entity.setTags(fields.contains("tags") ? getTags(entity.getFullyQualifiedName()) : null);
entity.setUsageCount(fields.contains("usageCount") ? getUsageCount(entity) : null);
return entity;
}
private Integer getUsageCount(GlossaryTerm term) {
return daoCollection.tagDAO().getTagCount(Source.GLOSSARY.ordinal(), term.getFullyQualifiedName());
}
private EntityReference getParent(GlossaryTerm entity) throws IOException {
List<String> ids =
findFrom(
@ -121,7 +127,7 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
EntityUtil.populateEntityReferences(entity.getReviewers());
// Set tags
entity.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), entity.getTags()));
entity.setTags(addDerivedTags(entity.getTags()));
}
@Override

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java
View File

@ -185,7 +185,7 @@ public class LocationRepository extends EntityRepository<Location> {
location.setServiceType(storageService.getServiceType());
location.setFullyQualifiedName(getFQN(location));
EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), location.getOwner()); // Validate owner
location.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), location.getTags()));
location.setTags(addDerivedTags(location.getTags()));
}
@Override

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MetricsRepository.java
View File

@ -75,7 +75,7 @@ public class MetricsRepository extends EntityRepository<Metrics> {
metrics.setFullyQualifiedName(getFQN(metrics));
EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), metrics.getOwner()); // Validate owner
metrics.setService(getService(metrics.getService()));
metrics.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), metrics.getTags()));
metrics.setTags(addDerivedTags(metrics.getTags()));
}
@Override

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java
View File

@ -152,7 +152,7 @@ public class MlModelRepository extends EntityRepository<MlModel> {
daoCollection.dashboardDAO().findEntityReferenceById(mlModel.getDashboard().getId());
}
mlModel.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), mlModel.getTags()));
mlModel.setTags(addDerivedTags(mlModel.getTags()));
}
@Override

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java
View File

@ -168,7 +168,7 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
populateService(pipeline);
pipeline.setFullyQualifiedName(getFQN(pipeline));
EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), pipeline.getOwner()); // Validate owner
pipeline.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), pipeline.getTags()));
pipeline.setTags(addDerivedTags(pipeline.getTags()));
}
@Override

16
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java
View File

@ -443,15 +443,15 @@ public class TableRepository extends EntityRepository<Table> {
});
}
private void addDerivedTags(List<Column> columns) throws IOException {
private void addDerivedColumnTags(List<Column> columns) throws IOException {
if (columns == null || columns.isEmpty()) {
return;
}
for (Column column : columns) {
column.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), column.getTags()));
column.setTags(addDerivedTags(column.getTags()));
if (column.getChildren() != null) {
addDerivedTags(column.getChildren());
addDerivedColumnTags(column.getChildren());
}
}
}
@ -475,10 +475,10 @@ public class TableRepository extends EntityRepository<Table> {
table.setOwner(helper(table).validateOwnerOrNull());
// Validate table tags and add derived tags to the list
table.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), table.getTags()));
table.setTags(addDerivedTags(table.getTags()));
// Validate column tags
addDerivedTags(table.getColumns());
addDerivedColumnTags(table.getColumns());
}
private EntityReference getDatabase(Table table) throws IOException, ParseException {
@ -564,7 +564,7 @@ public class TableRepository extends EntityRepository<Table> {
private void applyTags(List<Column> columns) {
// Add column level tags by adding tag to column relationship
for (Column column : columns) {
EntityUtil.applyTags(daoCollection.tagDAO(), column.getTags(), column.getFullyQualifiedName());
applyTags(column.getTags(), column.getFullyQualifiedName());
if (column.getChildren() != null) {
applyTags(column.getChildren());
}
@ -1023,11 +1023,11 @@ public class TableRepository extends EntityRepository<Table> {
}
// Delete tags related to deleted columns
deletedColumns.forEach(deleted -> EntityUtil.removeTags(daoCollection.tagDAO(), deleted.getFullyQualifiedName()));
deletedColumns.forEach(deleted -> daoCollection.tagDAO().deleteTags(deleted.getFullyQualifiedName()));
// Add tags related to newly added columns
for (Column added : addedColumns) {
EntityUtil.applyTags(daoCollection.tagDAO(), added.getTags(), added.getFullyQualifiedName());
applyTags(added.getTags(), added.getFullyQualifiedName());
}
// Carry forward the user generated metadata from existing columns to new columns

80
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TagDAO.java
View File

@ -1,80 +0,0 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import java.util.List;
import org.jdbi.v3.sqlobject.config.RegisterRowMapper;
import org.jdbi.v3.sqlobject.customizer.Bind;
import org.jdbi.v3.sqlobject.statement.SqlQuery;
import org.jdbi.v3.sqlobject.statement.SqlUpdate;
import org.openmetadata.catalog.jdbi3.TagRepository.TagLabelMapper;
import org.openmetadata.catalog.type.TagLabel;
/**
* Tag categories are stored as JSON in {@code tag_category} table. All the attributes are stored as JSON document
* except href, usageCount and children tags which are constructed on the fly as needed.
*
* <p>Tags are stored as JSON in {@code tag} table. All the attributes of tags are stored as JSON document except href,
* usageCount and children tags which are constructed on the fly as needed.
*/
@RegisterRowMapper(TagLabelMapper.class)
public interface TagDAO {
@SqlUpdate("INSERT INTO tag_category (json) VALUES (:json)")
void insertCategory(@Bind("json") String json);
@SqlUpdate("INSERT INTO tag(json) VALUES (:json)")
void insertTag(@Bind("json") String json);
@SqlUpdate("UPDATE tag_category SET json = :json where name = :name")
void updateCategory(@Bind("name") String name, @Bind("json") String json);
@SqlUpdate("UPDATE tag SET json = :json where fullyQualifiedName = :fqn")
void updateTag(@Bind("fqn") String fqn, @Bind("json") String json);
@SqlQuery("SELECT json FROM tag_category ORDER BY name")
List<String> listCategories();
@SqlQuery("SELECT json FROM tag WHERE fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') ORDER BY fullyQualifiedName")
List<String> listChildrenTags(@Bind("fqnPrefix") String fqnPrefix);
@SqlQuery("SELECT json FROM tag_category WHERE name = :name")
String findCategory(@Bind("name") String name);
@SqlQuery("SELECT EXISTS (SELECT * FROM tag WHERE fullyQualifiedName = :fqn)")
boolean tagExists(@Bind("fqn") String fqn);
@SqlQuery("SELECT json FROM tag WHERE fullyQualifiedName = :fqn")
String findTag(@Bind("fqn") String fqn);
@SqlUpdate(
"INSERT IGNORE INTO tag_usage (tagFQN, targetFQN, labelType, state) VALUES (:tagFQN, :targetFQN, "
+ ":labelType, :state)")
void applyTag(
@Bind("tagFQN") String tagFQN,
@Bind("targetFQN") String targetFQN,
@Bind("labelType") int labelType,
@Bind("state") int state);
@SqlQuery("SELECT tagFQN, labelType, state FROM tag_usage WHERE targetFQN = :targetFQN ORDER BY tagFQN")
List<TagLabel> getTags(@Bind("targetFQN") String targetFQN);
@SqlQuery("SELECT COUNT(*) FROM tag_usage WHERE tagFQN LIKE CONCAT(:fqnPrefix, '%')")
int getTagCount(@Bind("fqnPrefix") String fqnPrefix);
@SqlUpdate("DELETE FROM tag_usage where targetFQN = :targetFQN")
void deleteTags(@Bind("targetFQN") String targetFQN);
@SqlUpdate("DELETE FROM tag_usage where targetFQN LIKE CONCAT(:fqnPrefix, '%')")
void deleteTagsByPrefix(@Bind("fqnPrefix") String fqnPrefix);
}

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TagRepository.java
View File

@ -27,6 +27,7 @@ import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.type.Tag;
import org.openmetadata.catalog.type.TagCategory;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.TagLabel.Source;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.JsonUtils;
@ -251,11 +252,11 @@ public class TagRepository {
}
private Integer getUsageCount(TagCategory category) {
return dao.tagDAO().getTagCount(category.getName());
return dao.tagDAO().getTagCount(Source.TAG.ordinal(), category.getName());
}
private Integer getUsageCount(Tag tag) {
return dao.tagDAO().getTagCount(tag.getFullyQualifiedName());
return dao.tagDAO().getTagCount(Source.TAG.ordinal(), tag.getFullyQualifiedName());
}
public static class TagLabelMapper implements RowMapper<TagLabel> {

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java
View File

@ -75,7 +75,7 @@ public class TopicRepository extends EntityRepository<Topic> {
topic.setServiceType(messagingService.getServiceType());
topic.setFullyQualifiedName(getFQN(topic));
topic.setOwner(helper(topic).validateOwnerOrNull());
topic.setTags(EntityUtil.addDerivedTags(daoCollection.tagDAO(), topic.getTags()));
topic.setTags(addDerivedTags(topic.getTags()));
}
@Override

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java
View File

@ -111,7 +111,7 @@ public class GlossaryResource {
}
}
static final String FIELDS = "owner,tags,reviewers";
static final String FIELDS = "owner,tags,reviewers,usageCount";
public static final List<String> ALLOWED_FIELDS = Entity.getEntityFields(Glossary.class);
@GET

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResource.java
View File

@ -116,7 +116,7 @@ public class GlossaryTermResource {
}
}
static final String FIELDS = "children,relatedTerms,reviewers,tags";
static final String FIELDS = "children,relatedTerms,reviewers,tags,usageCount";
public static final List<String> ALLOWED_FIELDS = Entity.getEntityFields(GlossaryTerm.class);
@GET

71
catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java
View File

@ -15,7 +15,6 @@ package org.openmetadata.catalog.util;
import static org.openmetadata.catalog.type.Include.ALL;
import static org.openmetadata.catalog.type.Include.DELETED;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import java.io.IOException;
import java.util.ArrayList;
@ -44,7 +43,6 @@ import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityRelationshipDAO;
import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityVersionPair;
import org.openmetadata.catalog.jdbi3.CollectionDAO.TagDAO;
import org.openmetadata.catalog.jdbi3.CollectionDAO.TeamDAO;
import org.openmetadata.catalog.jdbi3.CollectionDAO.UsageDAO;
import org.openmetadata.catalog.jdbi3.CollectionDAO.UserDAO;
@ -62,9 +60,7 @@ import org.openmetadata.catalog.type.MlHyperParameter;
import org.openmetadata.catalog.type.Relationship;
import org.openmetadata.catalog.type.Schedule;
import org.openmetadata.catalog.type.TableConstraint;
import org.openmetadata.catalog.type.Tag;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.TagLabel.LabelType;
import org.openmetadata.catalog.type.Task;
import org.openmetadata.catalog.type.UsageDetails;
import org.openmetadata.catalog.type.UsageStats;
@ -279,73 +275,6 @@ public final class EntityUtil {
return details;
}
/** Apply tags {@code tagLabels} to the entity or field identified by {@code targetFQN} */
public static void applyTags(TagDAO tagDAO, List<TagLabel> tagLabels, String targetFQN) {
for (TagLabel tagLabel : listOrEmpty(tagLabels)) {
String json = tagDAO.findTag(tagLabel.getTagFQN());
if (json == null) {
// Invalid TagLabel
throw EntityNotFoundException.byMessage(
CatalogExceptionMessage.entityNotFound(Tag.class.getSimpleName(), tagLabel.getTagFQN()));
}
// Apply tagLabel to targetFQN that identifies an entity or field
tagDAO.applyTag(
tagLabel.getTagFQN(), targetFQN, tagLabel.getLabelType().ordinal(), tagLabel.getState().ordinal());
}
}
public static List<TagLabel> getDerivedTags(TagDAO tagDAO, TagLabel tagLabel, Tag tag) throws IOException {
List<TagLabel> derivedTags = new ArrayList<>();
for (String fqn : listOrEmpty(tag.getAssociatedTags())) {
String json = tagDAO.findTag(fqn);
if (json == null) {
// Invalid TagLabel
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Tag.class.getSimpleName(), fqn));
}
Tag tempTag = JsonUtils.readValue(json, Tag.class);
derivedTags.add(
new TagLabel()
.withTagFQN(fqn)
.withState(tagLabel.getState())
.withDescription(tempTag.getDescription())
.withLabelType(LabelType.DERIVED));
}
return derivedTags;
}
/** Validate given list of tags and add derived tags to it */
public static List<TagLabel> addDerivedTags(TagDAO tagDAO, List<TagLabel> tagLabels) throws IOException {
if (tagLabels == null || tagLabels.isEmpty()) {
return tagLabels;
}
List<TagLabel> updatedTagLabels = new ArrayList<>(tagLabels);
for (TagLabel tagLabel : tagLabels) {
String json = tagDAO.findTag(tagLabel.getTagFQN());
if (json == null) {
// Invalid TagLabel
throw EntityNotFoundException.byMessage(
CatalogExceptionMessage.entityNotFound(Tag.class.getSimpleName(), tagLabel.getTagFQN()));
}
Tag tag = JsonUtils.readValue(json, Tag.class);
// Apply derived tags
List<TagLabel> derivedTags = getDerivedTags(tagDAO, tagLabel, tag);
mergeTags(updatedTagLabels, derivedTags);
}
updatedTagLabels.sort(compareTagLabel);
return updatedTagLabels;
}
public static void removeTags(TagDAO tagDAO, String fullyQualifiedName) {
tagDAO.deleteTags(fullyQualifiedName);
}
public static void removeTagsByPrefix(TagDAO tagDAO, String fullyQualifiedName) {
tagDAO.deleteTagsByPrefix(fullyQualifiedName);
}
/** Merge derivedTags into tags, if it already does not exist in tags */
public static void mergeTags(List<TagLabel> tags, List<TagLabel> derivedTags) {
if (derivedTags == null || derivedTags.isEmpty()) {

4
catalog-rest-service/src/main/resources/json/schema/entity/data/glossary.json
View File

@ -57,6 +57,10 @@
"description": "Owner of this glossary.",
"$ref": "../../type/entityReference.json"
},
"usageCount": {
"description": "Count of how many times terms from this glossary are used.",
"type": "integer"
},
"tags": {
"description": "Tags for this glossary.",
"type": "array",

4
catalog-rest-service/src/main/resources/json/schema/entity/data/glossaryTerm.json
View File

@ -105,6 +105,10 @@
"description": "User names of the reviewers for this glossary.",
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList"
},
"usageCount": {
"description": "Count of how many times this and it's children glossary terms are used as labels.",
"type": "integer"
},
"tags": {
"description": "Tags for this glossary term.",
"type": "array",

8
catalog-rest-service/src/main/resources/json/schema/type/tagLabel.json
View File

@ -19,6 +19,12 @@
"description": "Unique name of the tag category.",
"type": "string"
},
"source": {
"description": "Label is from Tags or Glossary",
"type": "string",
"enum": ["Tag", "Glossary"],
"default": "Tag"
},
"labelType": {
"description": "Label type describes how a tag label was applied. 'Manual' indicates the tag label was applied by a person. 'Derived' indicates a tag label was derived using the associated tag relationship (see TagCategory.json for more details). 'Propagated` indicates a tag label was propagated from upstream based on lineage. 'Automated' is used when a tool was used to determine the tag label.",
"type": "string",
@ -36,6 +42,6 @@
"$ref": "basic.json#/definitions/href"
}
},
"required": ["tagFQN", "labelType", "state"],
"required": ["tagFQN", "source", "labelType", "state"],
"additionalProperties": false
}

20
catalog-rest-service/src/test/java/org/openmetadata/catalog/EnumBackwardCompatibilityTest.java
View File

@ -19,16 +19,18 @@ import org.junit.jupiter.api.Test;
import org.openmetadata.catalog.type.Relationship;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.TagLabel.LabelType;
import org.openmetadata.catalog.type.TagLabel.Source;
import org.openmetadata.catalog.type.TagLabel.State;
/**
* Enum ordinal number is stored in the database. New enums must be added at the end to ensure backward compatibility
*
* <p>Any time a new enum is added in the middle instead of at the end or enum ordinal value change, this test will
* fail. Update the test with total number of enums and test the ordinal number of the last enum. This will help catch
* new enum inadvertently being added in the middle.
*/
class EnumBackwardCompatibilityTest {
/**
* Any time a new enum is added, this test will fail. Update the test with total number of enums and test the ordinal
* number of the last enum. This will help catch new enum inadvertently being added in the middle.
*/
/** */
@Test
void testRelationshipEnumBackwardCompatible() {
assertEquals(17, Relationship.values().length);
@ -54,4 +56,14 @@ class EnumBackwardCompatibilityTest {
assertEquals(2, TagLabel.State.values().length);
assertEquals(1, State.CONFIRMED.ordinal());
}
/**
* Any time a new enum is added, this test will fail. Update the test with total number of enums and test the ordinal
* number of the last enum. This will help catch new enum inadvertently being added in the middle.
*/
@Test
void testTagSourceEnumBackwardCompatible() {
assertEquals(0, Source.TAG.ordinal());
assertEquals(1, Source.GLOSSARY.ordinal());
}
}

62
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java
View File

@ -80,6 +80,8 @@ import org.openmetadata.catalog.CatalogApplicationTest;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateChart;
import org.openmetadata.catalog.api.data.CreateDatabase;
import org.openmetadata.catalog.api.data.CreateGlossary;
import org.openmetadata.catalog.api.data.CreateGlossaryTerm;
import org.openmetadata.catalog.api.data.TermReference;
import org.openmetadata.catalog.api.services.CreateDashboardService;
import org.openmetadata.catalog.api.services.CreateDashboardService.DashboardServiceType;
@ -92,6 +94,8 @@ import org.openmetadata.catalog.api.services.CreatePipelineService.PipelineServi
import org.openmetadata.catalog.api.services.CreateStorageService;
import org.openmetadata.catalog.entity.data.Chart;
import org.openmetadata.catalog.entity.data.Database;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.entity.data.GlossaryTerm;
import org.openmetadata.catalog.entity.services.DashboardService;
import org.openmetadata.catalog.entity.services.DatabaseService;
import org.openmetadata.catalog.entity.services.MessagingService;
@ -104,6 +108,8 @@ import org.openmetadata.catalog.jdbi3.ChartRepository.ChartEntityInterface;
import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface;
import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface;
import org.openmetadata.catalog.jdbi3.GlossaryTermRepository.GlossaryTermEntityInterface;
import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.PipelineServiceRepository.PipelineServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.RoleRepository.RoleEntityInterface;
@ -114,6 +120,8 @@ import org.openmetadata.catalog.resources.charts.ChartResourceTest;
import org.openmetadata.catalog.resources.databases.DatabaseResourceTest;
import org.openmetadata.catalog.resources.events.EventResource.ChangeEventList;
import org.openmetadata.catalog.resources.events.WebhookResourceTest;
import org.openmetadata.catalog.resources.glossary.GlossaryResourceTest;
import org.openmetadata.catalog.resources.glossary.GlossaryTermResourceTest;
import org.openmetadata.catalog.resources.services.DashboardServiceResourceTest;
import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest;
import org.openmetadata.catalog.resources.services.MessagingServiceResourceTest;
@ -136,6 +144,7 @@ import org.openmetadata.catalog.type.Include;
import org.openmetadata.catalog.type.StorageServiceType;
import org.openmetadata.catalog.type.Tag;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.TagLabel.Source;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.JsonUtils;
@ -192,12 +201,24 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
public static EntityReference GCP_STORAGE_SERVICE_REFERENCE;
public static TagLabel USER_ADDRESS_TAG_LABEL;
public static TagLabel USER_BANK_ACCOUNT_TAG_LABEL;
public static TagLabel PERSONAL_DATA_TAG_LABEL;
public static TagLabel PII_SENSITIVE_TAG_LABEL;
public static TagLabel TIER1_TAG_LABEL;
public static TagLabel TIER2_TAG_LABEL;
public static Glossary GLOSSARY1;
public static EntityReference GLOSSARY1_REF;
public static Glossary GLOSSARY2;
public static EntityReference GLOSSARY2_REF;
public static GlossaryTerm GLOSSARY1_TERM1;
public static EntityReference GLOSSARY1_TERM1_REF;
public static TagLabel GLOSSARY1_TERM1_LABEL;
public static GlossaryTerm GLOSSARY2_TERM1;
public static EntityReference GLOSSARY2_TERM1_REF;
public static TagLabel GLOSSARY2_TERM1_LABEL;
public static EntityReference SUPERSET_REFERENCE;
public static EntityReference LOOKER_REFERENCE;
public static List<EntityReference> CHART_REFERENCES;
@ -353,7 +374,6 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
GCP_STORAGE_SERVICE_REFERENCE = new StorageServiceEntityInterface(service).getEntityReference();
USER_ADDRESS_TAG_LABEL = getTagLabel("User.Address");
USER_BANK_ACCOUNT_TAG_LABEL = getTagLabel("User.BankAccount");
PERSONAL_DATA_TAG_LABEL = getTagLabel("PersonalData.Personal");
PII_SENSITIVE_TAG_LABEL = getTagLabel("PII.Sensitive");
TIER1_TAG_LABEL = getTagLabel("Tier.Tier1");
@ -386,11 +406,39 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
DATABASE = databaseResourceTest.createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
DATABASE_REFERENCE = new DatabaseEntityInterface(DATABASE).getEntityReference();
GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest();
CreateGlossary createGlossary = glossaryResourceTest.createRequest("g1", "", "", null);
GLOSSARY1 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
GLOSSARY1_REF = new GlossaryEntityInterface(GLOSSARY1).getEntityReference();
createGlossary = glossaryResourceTest.createRequest("g2", "", "", null);
GLOSSARY2 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
GLOSSARY2_REF = new GlossaryEntityInterface(GLOSSARY2).getEntityReference();
GlossaryTermResourceTest glossaryTermResourceTest = new GlossaryTermResourceTest();
CreateGlossaryTerm createGlossaryTerm =
glossaryTermResourceTest
.createRequest("g1t1", null, "", null)
.withRelatedTerms(null)
.withGlossary(GLOSSARY1_REF);
GLOSSARY1_TERM1 = glossaryTermResourceTest.createEntity(createGlossaryTerm, ADMIN_AUTH_HEADERS);
GLOSSARY1_TERM1_REF = new GlossaryTermEntityInterface(GLOSSARY1_TERM1).getEntityReference();
GLOSSARY1_TERM1_LABEL = getTagLabel(GLOSSARY1_TERM1);
createGlossaryTerm =
glossaryTermResourceTest
.createRequest("g2t1", null, "", null)
.withRelatedTerms(null)
.withGlossary(GLOSSARY2_REF);
GLOSSARY2_TERM1 = glossaryTermResourceTest.createEntity(createGlossaryTerm, ADMIN_AUTH_HEADERS);
GLOSSARY2_TERM1_REF = new GlossaryTermEntityInterface(GLOSSARY2_TERM1).getEntityReference();
GLOSSARY2_TERM1_LABEL = getTagLabel(GLOSSARY2_TERM1);
COLUMNS =
Arrays.asList(
getColumn("c1", BIGINT, USER_ADDRESS_TAG_LABEL),
getColumn("c2", ColumnDataType.VARCHAR, USER_ADDRESS_TAG_LABEL).withDataLength(10),
getColumn("c3", BIGINT, USER_BANK_ACCOUNT_TAG_LABEL));
getColumn("c3", BIGINT, GLOSSARY1_TERM1_LABEL));
}
private TagLabel getTagLabel(String tagName) throws HttpResponseException {
@ -398,6 +446,13 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
return new TagLabel().withTagFQN(tag.getFullyQualifiedName()).withDescription(tag.getDescription());
}
private TagLabel getTagLabel(GlossaryTerm term) {
return new TagLabel()
.withTagFQN(term.getFullyQualifiedName())
.withDescription(term.getDescription())
.withSource(Source.GLOSSARY);
}
@AfterAll
public void afterAllTests() throws Exception {
if (runWebhookTests) {
@ -1141,6 +1196,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
if (supportsTags) {
entityInterface.setTags(new ArrayList<>());
entityInterface.getTags().add(USER_ADDRESS_TAG_LABEL);
entityInterface.getTags().add(GLOSSARY2_TERM1_LABEL);
change.getFieldsAdded().add(new FieldChange().withName("tags").withNewValue(entityInterface.getTags()));
}
if (supportsDots) {

101
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java
View File

@ -83,6 +83,8 @@ import org.openmetadata.catalog.entity.services.DatabaseService;
import org.openmetadata.catalog.jdbi3.TableRepository.TableEntityInterface;
import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.resources.databases.TableResource.TableList;
import org.openmetadata.catalog.resources.glossary.GlossaryResourceTest;
import org.openmetadata.catalog.resources.glossary.GlossaryTermResourceTest;
import org.openmetadata.catalog.resources.locations.LocationResourceTest;
import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest;
import org.openmetadata.catalog.resources.tags.TagResourceTest;
@ -253,7 +255,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
getColumn("column3", STRING, null)
.withOrdinalPosition(3)
.withDescription("column3")
.withTags(List.of(USER_ADDRESS_TAG_LABEL, USER_BANK_ACCOUNT_TAG_LABEL));
.withTags(List.of(USER_ADDRESS_TAG_LABEL, GLOSSARY1_TERM1_LABEL));
columns = new ArrayList<>();
columns.add(updateColumn1);
columns.add(updateColumn2);
@ -330,7 +332,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
// Column struct<a: int, b:char, c: struct<int: d>>>
Column c2 =
getColumn("c2", STRUCT, "struct<a: int, b:string, c: struct<int: d>>", USER_BANK_ACCOUNT_TAG_LABEL)
getColumn("c2", STRUCT, "struct<a: int, b:string, c: struct<int: d>>", GLOSSARY1_TERM1_LABEL)
.withChildren(new ArrayList<>(Arrays.asList(c2_a, c2_b, c2_c)));
// Test POST operation can create complex types
@ -364,10 +366,10 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
// struct<a:int, b:char, c:struct<d:int>>>
// to
// struct<-----, b:char, c:struct<d:int, e:char>, f:char>
c2_b.withTags(List.of(USER_ADDRESS_TAG_LABEL, USER_BANK_ACCOUNT_TAG_LABEL)); // Add new tag to c2.b tag
c2_b.withTags(List.of(USER_ADDRESS_TAG_LABEL, GLOSSARY1_TERM1_LABEL)); // Add new tag to c2.b tag
change
.getFieldsAdded()
.add(new FieldChange().withName("columns.c2.b.tags").withNewValue(List.of(USER_BANK_ACCOUNT_TAG_LABEL)));
.add(new FieldChange().withName("columns.c2.b.tags").withNewValue(List.of(GLOSSARY1_TERM1_LABEL)));
Column c2_c_e = getColumn("e", INT, USER_ADDRESS_TAG_LABEL);
c2_c.getChildren().add(c2_c_e); // Add c2.c.e
@ -401,13 +403,13 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
//
String tableJson = JsonUtils.pojoToJson(table1);
c1 = table1.getColumns().get(0);
c1.withTags(singletonList(USER_BANK_ACCOUNT_TAG_LABEL)); // c1 tag changed
c1.withTags(singletonList(GLOSSARY1_TERM1_LABEL)); // c1 tag changed
c2 = table1.getColumns().get(1);
c2.withTags(Arrays.asList(USER_ADDRESS_TAG_LABEL, USER_BANK_ACCOUNT_TAG_LABEL)); // c2 new tag added
c2.withTags(Arrays.asList(USER_ADDRESS_TAG_LABEL, GLOSSARY1_TERM1_LABEL)); // c2 new tag added
c2_a = c2.getChildren().get(0);
c2_a.withTags(singletonList(USER_BANK_ACCOUNT_TAG_LABEL)); // c2.a tag changed
c2_a.withTags(singletonList(GLOSSARY1_TERM1_LABEL)); // c2.a tag changed
c2_b = c2.getChildren().get(1);
c2_b.withTags(new ArrayList<>()); // c2.b tag removed
@ -416,7 +418,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
c2_c.withTags(new ArrayList<>()); // c2.c tag removed
c2_c_d = c2_c.getChildren().get(0);
c2_c_d.setTags(singletonList(USER_BANK_ACCOUNT_TAG_LABEL)); // c2.c.d new tag added
c2_c_d.setTags(singletonList(GLOSSARY1_TERM1_LABEL)); // c2.c.d new tag added
table1 = patchEntity(table1.getId(), tableJson, table1, ADMIN_AUTH_HEADERS);
assertColumns(Arrays.asList(c1, c2), table1.getColumns());
}
@ -517,7 +519,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
void put_updateColumns_200(TestInfo test) throws IOException {
int tagCategoryUsageCount = getTagCategoryUsageCount("user", TEST_AUTH_HEADERS);
int addressTagUsageCount = getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS);
int bankTagUsageCount = getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS);
int glossaryTermUsageCount = getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), TEST_AUTH_HEADERS);
//
// Create a table with column c1, type BIGINT, description c1 and tag USER_ADDRESS_TAB_LABEL
@ -534,25 +536,27 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
// Ensure tag category and tag usage counts are updated
assertEquals(tagCategoryUsageCount + 1, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS));
assertEquals(addressTagUsageCount + 1, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS));
assertEquals(bankTagUsageCount, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS));
assertEquals(
glossaryTermUsageCount, getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), TEST_AUTH_HEADERS));
//
// Update the c1 tags to USER_ADDRESS_TAB_LABEL, USER_BANK_ACCOUNT_TAG_LABEL (newly added)
// Update the c1 tags to USER_ADDRESS_TAB_LABEL, GLOSSARY1_TERM1_LABEL (newly added)
// Ensure description and previous tag is carried forward during update
//
tags.add(USER_BANK_ACCOUNT_TAG_LABEL);
tags.add(GLOSSARY1_TERM1_LABEL);
List<Column> updatedColumns = new ArrayList<>();
updatedColumns.add(getColumn("c1", BIGINT, null).withTags(tags));
ChangeDescription change = getChangeDescription(table.getVersion());
change
.getFieldsAdded()
.add(new FieldChange().withName("columns.c1.tags").withNewValue(List.of(USER_BANK_ACCOUNT_TAG_LABEL)));
.add(new FieldChange().withName("columns.c1.tags").withNewValue(List.of(GLOSSARY1_TERM1_LABEL)));
table = updateAndCheckEntity(request.withColumns(updatedColumns), OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
// Ensure tag usage counts are updated
assertEquals(tagCategoryUsageCount + 2, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS));
assertEquals(tagCategoryUsageCount + 1, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS));
assertEquals(addressTagUsageCount + 1, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS));
assertEquals(bankTagUsageCount + 1, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS));
assertEquals(
glossaryTermUsageCount + 1, getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), TEST_AUTH_HEADERS));
//
// Add a new column c2 using PUT
@ -563,10 +567,11 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
change.getFieldsAdded().add(new FieldChange().withName("columns").withNewValue(List.of(c2)));
table = updateAndCheckEntity(request.withColumns(updatedColumns), OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
// Ensure tag usage counts are updated - column c2 added both address and bank tags
assertEquals(tagCategoryUsageCount + 4, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS));
// Ensure tag usage counts are updated - column c2 added both address
assertEquals(tagCategoryUsageCount + 2, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS));
assertEquals(addressTagUsageCount + 2, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS));
assertEquals(bankTagUsageCount + 2, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS));
assertEquals(
glossaryTermUsageCount + 2, getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), TEST_AUTH_HEADERS));
//
// Change the column c2 data length from 10 to 20. Increasing the data length is considered backward compatible
@ -597,9 +602,10 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
assertEquals(1, table.getColumns().size());
// Ensure tag usage counts are updated to reflect removal of column c2
assertEquals(tagCategoryUsageCount + 2, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS));
assertEquals(tagCategoryUsageCount + 1, getTagCategoryUsageCount("user", TEST_AUTH_HEADERS));
assertEquals(addressTagUsageCount + 1, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS));
assertEquals(bankTagUsageCount + 1, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), TEST_AUTH_HEADERS));
assertEquals(
glossaryTermUsageCount + 1, getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), TEST_AUTH_HEADERS));
}
@Test
@ -1213,16 +1219,25 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
CreateTable create =
createRequest(test, 1)
.withOwner(USER_OWNER1)
.withTags(singletonList(USER_ADDRESS_TAG_LABEL)) // 1 table tag - USER_ADDRESS
.withColumns(COLUMNS); // 3 column tags - 2 USER_ADDRESS and 1 USER_BANK_ACCOUNT
.withTags(List.of(USER_ADDRESS_TAG_LABEL, GLOSSARY2_TERM1_LABEL)) // 2 table tags - USER_ADDRESS, g2t1
.withColumns(COLUMNS); // 3 column tags - 2 USER_ADDRESS and 1 g1t1
createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
// Total 4 user tags - 1 table 1 tag + 3 column tags
assertEquals(4, getTagCategoryUsageCount("user", ADMIN_AUTH_HEADERS));
// Total 3 USER_ADDRESS tags - 1 table 1 tag and 2 column tags
// Total 3 user tags - 1 table tag + 2 column tags
assertEquals(3, getTagCategoryUsageCount("user", ADMIN_AUTH_HEADERS));
// Total 1 glossary1 tags - 1 column
assertEquals(1, getGlossaryUsageCount("g1", ADMIN_AUTH_HEADERS));
// Total 1 glossary2 tags - 1 table
assertEquals(1, getGlossaryUsageCount("g2", ADMIN_AUTH_HEADERS));
// Total 3 USER_ADDRESS tags - 1 table tag and 2 column tags
assertEquals(3, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS));
// Total 1 USER_BANK_ACCOUNT tags - 1 column level
assertEquals(1, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS));
// Total 1 GLOSSARY1_TERM1 - 1 column level
assertEquals(1, getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS));
// Total 1 GLOSSARY1_TERM1 - 1 table level
assertEquals(1, getGlossaryTermUsageCount(GLOSSARY2_TERM1_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS));
// Create a table test2 with 3 column tags
CreateTable create1 =
@ -1232,12 +1247,12 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
.withColumns(COLUMNS); // 3 column tags - 2 USER_ADDRESS and 1 USER_BANK_ACCOUNT
createAndCheckEntity(create1, ADMIN_AUTH_HEADERS);
// Additional 3 user tags - 3 column tags
assertEquals(7, getTagCategoryUsageCount("user", ADMIN_AUTH_HEADERS));
// Additional 2 user tags - 2 column tags
assertEquals(5, getTagCategoryUsageCount("user", ADMIN_AUTH_HEADERS));
// Additional 2 USER_ADDRESS tags - 2 column tags
assertEquals(5, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS));
// Additional 2 USER_BANK_ACCOUNT tags - 1 column tags
assertEquals(2, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS));
// Additional 1 glossary tag - 1 column tags
assertEquals(2, getGlossaryTermUsageCount(GLOSSARY1_TERM1_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS));
ResultList<Table> tableList = listEntities(null, ADMIN_AUTH_HEADERS); // List tables
assertEquals(2, tableList.getData().size());
@ -1372,7 +1387,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
List<Column> columns = new ArrayList<>();
columns.add(getColumn("c1", INT, USER_ADDRESS_TAG_LABEL).withDescription(null));
columns.add(getColumn("c2", BIGINT, USER_ADDRESS_TAG_LABEL));
columns.add(getColumn("c3", FLOAT, USER_BANK_ACCOUNT_TAG_LABEL));
columns.add(getColumn("c3", FLOAT, GLOSSARY1_TERM1_LABEL));
Table table = createEntity(createRequest(test).withColumns(columns), ADMIN_AUTH_HEADERS);
@ -1381,7 +1396,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
columns
.get(0)
.withDescription("new0") // Set new description
.withTags(List.of(USER_ADDRESS_TAG_LABEL, USER_BANK_ACCOUNT_TAG_LABEL));
.withTags(List.of(USER_ADDRESS_TAG_LABEL, GLOSSARY1_TERM1_LABEL));
change
.getFieldsAdded()
.add(
@ -1391,7 +1406,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
.add(
new FieldChange()
.withName("columns.c1.tags")
.withNewValue(List.of(USER_BANK_ACCOUNT_TAG_LABEL))); // Column c1 got new tags
.withNewValue(List.of(GLOSSARY1_TERM1_LABEL))); // Column c1 got new tags
columns
.get(1)
@ -1411,7 +1426,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
.add(
new FieldChange()
.withName("columns.c3.tags")
.withOldValue(List.of(USER_BANK_ACCOUNT_TAG_LABEL))); // Column c3 tags were removed
.withOldValue(List.of(GLOSSARY1_TERM1_LABEL))); // Column c3 tags were removed
String originalJson = JsonUtils.pojoToJson(table);
table.setColumns(columns);
@ -1705,6 +1720,15 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
return TagResourceTest.getCategory(name, "usageCount", authHeaders).getUsageCount();
}
private static int getGlossaryUsageCount(String name, Map<String, String> authHeaders) throws HttpResponseException {
return new GlossaryResourceTest().getEntityByName(name, "usageCount", authHeaders).getUsageCount();
}
private static int getGlossaryTermUsageCount(String name, Map<String, String> authHeaders)
throws HttpResponseException {
return new GlossaryTermResourceTest().getEntityByName(name, "usageCount", authHeaders).getUsageCount();
}
private void verifyTableProfileData(List<TableProfile> actualProfiles, List<TableProfile> expectedProfiles) {
assertEquals(actualProfiles.size(), expectedProfiles.size());
Map<String, TableProfile> tableProfileMap = new HashMap<>();
@ -1718,8 +1742,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
}
}
private void verifyTableTest(String tableName, List<TableTest> actualTests, List<CreateTableTest> expectedTests)
throws IOException {
private void verifyTableTest(String tableName, List<TableTest> actualTests, List<CreateTableTest> expectedTests) {
assertEquals(expectedTests.size(), actualTests.size());
Map<String, TableTest> tableTestMap = new HashMap<>();
for (TableTest test : actualTests) {
@ -1759,7 +1782,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
}
}
private void verifyColumnTest(Table table, Column column, List<CreateColumnTest> expectedTests) throws IOException {
private void verifyColumnTest(Table table, Column column, List<CreateColumnTest> expectedTests) {
List<ColumnTest> actualTests = new ArrayList<>();
for (Column c : table.getColumns()) {
if (c.getName().equals(column.getName())) {
@ -1818,7 +1841,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
}
}
private void verifyTestCaseResults(TestCaseResult expected, List<TestCaseResult> actual) throws IOException {
private void verifyTestCaseResults(TestCaseResult expected, List<TestCaseResult> actual) {
Map<Long, TestCaseResult> actualResultMap = new HashMap<>();
for (Object a : actual) {
TestCaseResult result = JsonUtils.convertValue(a, TestCaseResult.class);

39
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResourceTest.java
View File

@ -30,14 +30,12 @@ import static org.openmetadata.catalog.util.TestUtils.validateEntityReference;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
@ -64,16 +62,6 @@ import org.openmetadata.catalog.util.TestUtils.UpdateType;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, CreateGlossaryTerm> {
public static Glossary GLOSSARY1;
public static EntityReference GLOSSARY_REF1;
public static Glossary GLOSSARY2;
public static EntityReference GLOSSARY_REF2;
public static GlossaryTerm GLOSSARY_TERM1;
public static EntityReference GLOSSARY_TERM_REF1;
public static GlossaryTerm GLOSSARY_TERM2;
public static EntityReference GLOSSARY_TERM_REF2;
public GlossaryTermResourceTest() {
super(
Entity.GLOSSARY_TERM,
@ -88,27 +76,6 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
false);
}
@BeforeAll
public void setup(TestInfo test) throws IOException, URISyntaxException {
super.setup(test);
GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest();
CreateGlossary createGlossary = glossaryResourceTest.createRequest(test, 1);
GLOSSARY1 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
GLOSSARY_REF1 = new GlossaryEntityInterface(GLOSSARY1).getEntityReference();
createGlossary = glossaryResourceTest.createRequest(test, 2);
GLOSSARY2 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
GLOSSARY_REF2 = new GlossaryEntityInterface(GLOSSARY2).getEntityReference();
CreateGlossaryTerm createGlossaryTerm = createRequest(test, 1).withRelatedTerms(null);
GLOSSARY_TERM1 = createEntity(createGlossaryTerm, ADMIN_AUTH_HEADERS);
GLOSSARY_TERM_REF1 = new GlossaryTermEntityInterface(GLOSSARY_TERM1).getEntityReference();
createGlossaryTerm = createRequest(test, 2).withRelatedTerms(null);
GLOSSARY_TERM2 = createEntity(createGlossaryTerm, ADMIN_AUTH_HEADERS);
GLOSSARY_TERM_REF2 = new GlossaryTermEntityInterface(GLOSSARY_TERM2).getEntityReference();
}
@Order(0)
@Test
void get_listGlossaryTermsWithDifferentFilters() throws HttpResponseException {
@ -150,7 +117,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
queryParams.put("fields", "children,relatedTerms,reviewers,tags");
ResultList<GlossaryTerm> list = listEntities(queryParams, ADMIN_AUTH_HEADERS);
List<GlossaryTerm> expectedTerms =
Arrays.asList(GLOSSARY_TERM1, GLOSSARY_TERM2, term1, term11, term12, term2, term21, term22);
Arrays.asList(GLOSSARY1_TERM1, GLOSSARY2_TERM1, term1, term11, term12, term2, term21, term22);
assertContains(expectedTerms, list.getData());
// List terms under glossary1
@ -262,8 +229,8 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
.withSynonyms(List.of("syn1", "syn2", "syn3"))
.withDescription(description)
.withDisplayName(displayName)
.withGlossary(GLOSSARY_REF1)
.withRelatedTerms(Arrays.asList(GLOSSARY_TERM_REF1, GLOSSARY_TERM_REF2))
.withGlossary(GLOSSARY1_REF)
.withRelatedTerms(Arrays.asList(GLOSSARY1_TERM1_REF, GLOSSARY2_TERM1_REF))
.withReviewers(List.of(USER_OWNER1));
}

4
catalog-rest-service/src/test/java/org/openmetadata/catalog/util/TestUtils.java
View File

@ -49,6 +49,7 @@ import org.openmetadata.catalog.type.DatabaseConnection;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.Tag;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.TagLabel.Source;
@Slf4j
public final class TestUtils {
@ -238,6 +239,9 @@ public final class TestUtils {
// So add to the expectedList, the derived tags before validating the tags
List<TagLabel> updatedExpectedList = new ArrayList<>(expectedList);
for (TagLabel expected : expectedList) {
if (expected.getSource() != Source.TAG) {
continue; // TODO similar test for glossary
}
Tag tag = TagResourceTest.getTag(expected.getTagFQN(), ADMIN_AUTH_HEADERS);
List<TagLabel> derived = new ArrayList<>();
for (String fqn : listOrEmpty(tag.getAssociatedTags())) {