Fixes #4255 Move associated tags functionality from Tags to GlossaryT… (#4256)

This commit is contained in:
Suresh Srinivas 2022-04-19 15:29:28 -07:00 committed by GitHub
parent 4a58530286
commit 14fce51749
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 150 additions and 254 deletions

View File

@ -1275,9 +1275,13 @@ public interface CollectionDAO {
@Bind("state") int state);
@SqlQuery(
"SELECT tu.source, tu.tagFQN, tu.labelType, tu.state, t.json ->> '$.description' "
+ "AS description FROM tag_usage tu "
+ "LEFT JOIN tag t ON tu.tagFQN = t.fullyQualifiedName WHERE tu.targetFQN = :targetFQN ORDER BY tu.tagFQN")
"SELECT tu.source, tu.tagFQN, tu.labelType, tu.state, "
+ "t.json ->> '$.description' AS description1, "
+ "g.json ->> '$.description' AS description2 "
+ "FROM tag_usage tu "
+ "LEFT JOIN tag t ON tu.tagFQN = t.fullyQualifiedName AND tu.source = 0 "
+ "LEFT JOIN glossary_term_entity g ON tu.tagFQN = g.fullyQualifiedName AND tu.source = 1 "
+ "WHERE tu.targetFQN = :targetFQN ORDER BY tu.tagFQN")
List<TagLabel> getTags(@Bind("targetFQN") String targetFQN);
@SqlQuery("SELECT COUNT(*) FROM tag_usage WHERE tagFQN LIKE CONCAT(:fqnPrefix, '%') AND source = :source")
@ -1295,12 +1299,14 @@ public interface CollectionDAO {
class TagLabelMapper implements RowMapper<TagLabel> {
@Override
public TagLabel map(ResultSet r, StatementContext ctx) throws SQLException {
String description1 = r.getString("description1");
String description2 = r.getString("description2");
return new TagLabel()
.withSource(TagLabel.Source.values()[r.getInt("source")])
.withLabelType(TagLabel.LabelType.values()[r.getInt("labelType")])
.withState(TagLabel.State.values()[r.getInt("state")])
.withTagFQN(r.getString("tagFQN"))
.withDescription(r.getString("description"));
.withDescription(description1 == null ? description2 : description1);
}
}
}

View File

@ -33,6 +33,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@ -46,6 +47,7 @@ import org.apache.maven.shared.utils.io.IOUtil;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.CatalogApplicationConfig;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.GlossaryTerm;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.entity.teams.User;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
@ -584,40 +586,22 @@ public abstract class EntityRepository<T> {
}
List<TagLabel> updatedTagLabels = new ArrayList<>();
EntityUtil.mergeTags(updatedTagLabels, tagLabels);
for (TagLabel tagLabel : tagLabels) {
TagLabel existingTag =
updatedTagLabels.stream().filter(c -> tagLabelMatch.test(c, tagLabel)).findAny().orElse(null);
if (existingTag != null) {
continue; // tag label is already seen. Don't add duplicate tags.
}
updatedTagLabels.add(tagLabel);
if (tagLabel.getSource() != Source.TAG) {
continue; // Related tags are not supported for Glossary yet
}
Tag tag = daoCollection.tagDAO().findEntityByName(tagLabel.getTagFQN());
// Apply derived tags
List<TagLabel> derivedTags = getDerivedTags(tagLabel, tag);
EntityUtil.mergeTags(updatedTagLabels, derivedTags);
EntityUtil.mergeTags(updatedTagLabels, getDerivedTags(tagLabel));
}
updatedTagLabels.sort(compareTagLabel);
return updatedTagLabels;
}
/** Get tags associated with a given set of tags */
private List<TagLabel> getDerivedTags(TagLabel tagLabel, Tag tag) {
List<TagLabel> derivedTags = new ArrayList<>();
for (String fqn : listOrEmpty(tag.getAssociatedTags())) {
Tag tempTag = daoCollection.tagDAO().findEntityByName(fqn);
derivedTags.add(
new TagLabel()
.withTagFQN(fqn)
.withState(tagLabel.getState())
.withDescription(tempTag.getDescription())
.withLabelType(LabelType.DERIVED));
private List<TagLabel> getDerivedTags(TagLabel tagLabel) {
if (tagLabel.getSource() == Source.GLOSSARY) { // Related tags are only supported for Glossary
List<TagLabel> derivedTags = daoCollection.tagUsageDAO().getTags(tagLabel.getTagFQN());
derivedTags.forEach(tag -> tag.setLabelType(LabelType.DERIVED));
return derivedTags;
}
return derivedTags;
return Collections.emptyList();
}
protected void applyTags(T entity) {
@ -625,8 +609,6 @@ public abstract class EntityRepository<T> {
// Add entity level tags by adding tag to the entity relationship
EntityInterface<T> entityInterface = getEntityInterface(entity);
applyTags(entityInterface.getTags(), entityInterface.getFullyQualifiedName());
// Update tag to handle additional derived tags
entityInterface.setTags(getTags(entityInterface.getFullyQualifiedName()));
}
}
@ -634,9 +616,13 @@ public abstract class EntityRepository<T> {
public void applyTags(List<TagLabel> tagLabels, String targetFQN) {
for (TagLabel tagLabel : listOrEmpty(tagLabels)) {
if (tagLabel.getSource() == Source.TAG) {
daoCollection.tagDAO().findEntityByName(tagLabel.getTagFQN());
Tag tag = daoCollection.tagDAO().findEntityByName(tagLabel.getTagFQN());
tagLabel.withDescription(tag.getDescription());
tagLabel.setSource(Source.TAG);
} else if (tagLabel.getSource() == Source.GLOSSARY) {
daoCollection.glossaryTermDAO().findEntityByName(tagLabel.getTagFQN(), Include.NON_DELETED);
GlossaryTerm term = daoCollection.glossaryTermDAO().findEntityByName(tagLabel.getTagFQN(), Include.NON_DELETED);
tagLabel.withDescription(term.getDescription());
tagLabel.setSource(Source.GLOSSARY);
}
// Apply tagLabel to targetFQN that identifies an entity or field

View File

@ -15,7 +15,6 @@ package org.openmetadata.catalog.jdbi3;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
@ -266,21 +265,7 @@ public class FeedRepository {
return thread.getPosts();
}
/**
* List threads based on the filters and limits in the order of the updated timestamp.
*
* @param link entity link filter
* @param limitPosts the number of posts to limit per thread
* @param userId UUID of the user. Enables UserId filter
* @param filterType Type of the filter to be applied with userId filter
* @param limit the number of threads to limit in the response
* @param pageMarker the before/after updatedTime to be used as pagination marker for queries
* @param isResolved whether the thread is resolved or open
* @param paginationType before or after
* @return a list of threads as ResultList
* @throws IOException on error
* @throws ParseException on error
*/
/** List threads based on the filters and limits in the order of the updated timestamp. */
@Transaction
public final ResultList<Thread> list(
String link,
@ -435,12 +420,7 @@ public class FeedRepository {
return !original.getResolved().equals(updated.getResolved()) || !original.getMessage().equals(updated.getMessage());
}
/**
* Limit the number of posts within each thread.
*
* @param threads list of threads
* @param limitPosts the number of posts to limit per thread
*/
/** Limit the number of posts within each thread. */
private void limitPostsInThreads(List<Thread> threads, int limitPosts) {
for (Thread t : threads) {
List<Post> posts = t.getPosts();
@ -456,14 +436,6 @@ public class FeedRepository {
/**
* Return the threads associated with user/team owned entities and the threads that were created by or replied to by
* the user.
*
* @param userId UUID of the user
* @param limit number of threads to limit
* @param time updatedTime before/after which the results should be filtered for pagination
* @param isResolved whether the thread is resolved or open
* @param paginationType before or after
* @return a list of threads and the total count of threads
* @throws IOException on error
*/
private FilteredThreads getThreadsByOwner(
String userId, int limit, long time, boolean isResolved, PaginationType paginationType) throws IOException {
@ -481,12 +453,7 @@ public class FeedRepository {
return new FilteredThreads(threads, totalCount);
}
/**
* Get a list of team ids that the given user is a part of.
*
* @param userId UUID of the user.
* @return list of team ids.
*/
/** Get a list of team ids that the given user is a part of. */
private List<String> getTeamIds(String userId) {
List<String> teamIds = dao.relationshipDAO().findFrom(userId, Entity.USER, Relationship.HAS.ordinal(), Entity.TEAM);
if (teamIds.isEmpty()) {
@ -494,17 +461,7 @@ public class FeedRepository {
}
return teamIds;
}
/**
* Returns the threads where the user or the team they belong to were mentioned by other users with @mention.
*
* @param userId UUID of the user
* @param limit number of threads to limit
* @param time updatedTime before/after which the results should be filtered for pagination
* @param isResolved whether the thread is resolved or open
* @param paginationType before or after
* @return a list of threads and the total count of threads
* @throws IOException on error
*/
/** Returns the threads where the user or the team they belong to were mentioned by other users with @mention. */
private FilteredThreads getThreadsByMentions(
String userId, int limit, long time, boolean isResolved, PaginationType paginationType) throws IOException {
List<EntityReference> teams =
@ -536,17 +493,7 @@ public class FeedRepository {
return new FilteredThreads(threads, totalCount);
}
/**
* Returns the threads that are associated with the entities followed by the user.
*
* @param userId UUID of the user
* @param limit number of threads to limit
* @param time updatedTime before/after which the results should be filtered for pagination
* @param isResolved whether the thread is resolved or open
* @param paginationType before or after
* @return a list of threads and the total count of threads
* @throws IOException on error
*/
/** Returns the threads that are associated with the entities followed by the user. */
private FilteredThreads getThreadsByFollows(
String userId, int limit, long time, boolean isResolved, PaginationType paginationType) throws IOException {
List<String> jsons;

View File

@ -118,7 +118,7 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
// Validate reviewers
EntityUtil.populateEntityReferences(entity.getReviewers());
// Set tags
// Validate table tags and add derived tags to the list
entity.setTags(addDerivedTags(entity.getTags()));
}

View File

@ -523,7 +523,7 @@ public class TableRepository extends EntityRepository<Table> {
});
}
private void addDerivedColumnTags(List<Column> columns) throws IOException {
private void addDerivedColumnTags(List<Column> columns) {
if (columns == null || columns.isEmpty()) {
return;
}
@ -673,7 +673,7 @@ public class TableRepository extends EntityRepository<Table> {
}
}
private void validateColumnFQNs(List<JoinedWith> joinedWithList) throws IOException {
private void validateColumnFQNs(List<JoinedWith> joinedWithList) {
for (JoinedWith joinedWith : joinedWithList) {
// Validate table
String tableFQN = FullyQualifiedName.getTableFQN(joinedWith.getFullyQualifiedName());

View File

@ -23,9 +23,6 @@ public final class Migration {
/**
* Run a query to MySQL to retrieve the last migrated Flyway version. If the Flyway table DATABASE_CHANGE_LOG does not
* exist, we will stop the Catalog App and inform users how to run Flyway.
*
* @param jdbi JDBI connection
* @return Last migrated version, e.g., "003"
*/
public static Optional<String> lastMigrated(Jdbi jdbi) {
try {
@ -44,13 +41,7 @@ public final class Migration {
return Collections.max(migrationFiles);
}
/**
* Read the migrations path from the Catalog YAML config and return a list of all the files' versions.
*
* @param conf Catalog migration config
* @return List of migration files' versions
* @throws IOException If we cannot read the files
*/
/** Read the migrations path from the Catalog YAML config and return a list of all the files' versions. */
private static List<String> getMigrationVersions(MigrationConfiguration conf) throws IOException {
try (Stream<String> names =
Files.walk(Paths.get(conf.getPath()))
@ -63,12 +54,7 @@ public final class Migration {
}
}
/**
* Given a Flyway migration filename, e.g., v001__my_file.sql, return the version information "001".
*
* @param name Flyway migration filename
* @return File version
*/
/** Given a Flyway migration filename, e.g., v001__my_file.sql, return the version information "001". */
private static String cleanName(String name) {
return Arrays.asList(name.split("_")).get(0).replace("v", "");
}

View File

@ -525,7 +525,6 @@ public class TagResource {
.withName(create.getName())
.withFullyQualifiedName(FullyQualifiedName.add(parentFQN, create.getName()))
.withDescription(create.getDescription())
.withAssociatedTags(create.getAssociatedTags())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(System.currentTimeMillis());
}

View File

@ -129,14 +129,7 @@ public final class ChangeEventParser {
return fieldValue.toString();
}
/**
* Tries to merge additions and deletions into updates and returns a map of formatted messages.
*
* @param entity Entity object.
* @param addedFields Fields that were added as part of the change event.
* @param deletedFields Fields that were deleted as part of the change event.
* @return A map of entity link -> formatted message.
*/
/** Tries to merge additions and deletions into updates and returns a map of formatted messages. */
private static Map<EntityLink, String> getFormattedMessages(
Object entity, List<FieldChange> addedFields, List<FieldChange> deletedFields) {
// Major schema version changes such as renaming a column from colA to colB

View File

@ -82,7 +82,7 @@ public final class EntityUtil {
//
public static final BiPredicate<Object, Object> objectMatch = Object::equals;
public static final BiPredicate<EntityInterface, EntityInterface> entityMatch =
public static final BiPredicate<EntityInterface<?>, EntityInterface<?>> entityMatch =
(ref1, ref2) -> ref1.getId().equals(ref2.getId());
public static final BiPredicate<EntityReference, EntityReference> entityReferenceMatch =

View File

@ -18,7 +18,7 @@ import org.openmetadata.catalog.FqnParser.UnquotedNameContext;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
public class FullyQualifiedName {
// Quoted name of format "sss" or sss
// Quoted name of format "sss" or unquoted string sss
private static final Pattern namePattern = Pattern.compile("^(\")([^\"]+)(\")$|^(.*)$");
private FullyQualifiedName() {

View File

@ -37,6 +37,6 @@
"default": null
}
},
"required": ["name"],
"required": ["name", "description"],
"additionalProperties": false
}

View File

@ -56,6 +56,6 @@
"default": null
}
},
"required": ["glossary", "name"],
"required": ["glossary", "name", "description"],
"additionalProperties": false
}

View File

@ -77,6 +77,6 @@
"default": false
}
},
"required": ["id", "name"],
"required": ["id", "name", "description"],
"additionalProperties": false
}

View File

@ -106,7 +106,7 @@
"type": "integer"
},
"tags": {
"description": "Tags for this glossary term.",
"description": "Tags associated with this glossary term. These tags captures relationship of a glossary term with a tag automatically. As an example a glossary term 'User.PhoneNumber' might have an associated tag 'PII.Sensitive'. When 'User.Address' is used to label a column in a table, 'PII.Sensitive' label is also applied automatically due to Associated tag relationship.",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
@ -127,6 +127,6 @@
"default": false
}
},
"required": ["id", "name", "glossary"],
"required": ["id", "name", "description", "glossary"],
"additionalProperties": false
}

View File

@ -76,13 +76,6 @@
"type": "boolean",
"default": false
},
"associatedTags": {
"description": "Fully qualified names of tags associated with this tag. Associated tags captures relationship of one tag to another automatically. As an example a tag 'User.PhoneNumber' might have an associated tag 'PII.Sensitive'. When 'User.Address' is used to label a column in a table, 'PII.Sensitive' label is also applied automatically due to Associated tag relationship.",
"type": "array",
"items": {
"type": "string"
}
},
"children": {
"description": "Tags under this tag group or empty for tags at the leaf level.",
"type": "array",

View File

@ -143,6 +143,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
protected boolean supportsSoftDelete = true;
protected boolean supportsAuthorizedMetadataOperations = true;
protected boolean supportsFieldsQueryParam = true;
protected boolean supportsEmptyDescription = true;
public static final String DATA_STEWARD_ROLE_NAME = "DataSteward";
public static final String DATA_CONSUMER_ROLE_NAME = "DataConsumer";
@ -271,11 +272,11 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
// Create request such as CreateTable, CreateChart returned by concrete implementation
public K createRequest(TestInfo test) {
return createRequest(getEntityName(test), null, null, null);
return createRequest(getEntityName(test), "", null, null);
}
public K createRequest(TestInfo test, int index) {
return createRequest(getEntityName(test, index), null, null, null);
return createRequest(getEntityName(test, index), "", null, null);
}
public abstract K createRequest(String name, String description, String displayName, EntityReference owner);
@ -375,11 +376,11 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
List<UUID> createdUUIDs = new ArrayList<>();
for (int i = 0; i < maxEntities; i++) {
createdUUIDs.add(
getEntityInterface(createEntity(createRequest(getEntityName(test, i), null, null, null), ADMIN_AUTH_HEADERS))
getEntityInterface(createEntity(createRequest(getEntityName(test, i), "", null, null), ADMIN_AUTH_HEADERS))
.getId());
}
T entity = createEntity(createRequest(getEntityName(test, -1), null, null, null), ADMIN_AUTH_HEADERS);
T entity = createEntity(createRequest(getEntityName(test, -1), "", null, null), ADMIN_AUTH_HEADERS);
EntityInterface<T> deleted = getEntityInterface(entity);
deleteAndCheckEntity(entity, ADMIN_AUTH_HEADERS);
@ -709,19 +710,18 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
void post_entityWithDots_200() throws HttpResponseException {
// Entity without "." should not have quoted fullyQualifiedName
String name = String.format("%s_foo_bar", entityType);
K request = createRequest(name, null, null, null);
K request = createRequest(name, "", null, null);
T entity = createEntity(request, ADMIN_AUTH_HEADERS);
EntityInterface<T> entityInterface = getEntityInterface(entity);
assertFalse(entityInterface.getFullyQualifiedName().contains("\""));
// Now post entity name with dots. FullyQualifiedName must have " to escape dotted name
name = String.format("%s_foo.bar", entityType);
request = createRequest(name, null, null, null);
request = createRequest(name, "", null, null);
entity = createEntity(request, ADMIN_AUTH_HEADERS);
entityInterface = getEntityInterface(entity);
assertTrue(entityInterface.getFullyQualifiedName().contains("\""));
String[] split = FullyQualifiedName.split(entityInterface.getFullyQualifiedName());
System.out.println("XXX fqn is " + entityInterface.getFullyQualifiedName());
String actualName = split[split.length - 1];
assertEquals(name, entityInterface.getName());
assertEquals(FullyQualifiedName.quoteName(name), actualName);
@ -756,15 +756,15 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
return; // Entity doesn't support ownership
}
// Create a new entity with PUT as admin user
K request = createRequest(getEntityName(test), null, null, USER_OWNER1);
K request = createRequest(getEntityName(test), "", null, USER_OWNER1);
T entity = createAndCheckEntity(request, ADMIN_AUTH_HEADERS);
EntityInterface<T> entityInterface = getEntityInterface(entity);
// Update the entity as USER_OWNER1
request = createRequest(getEntityName(test), "newDescription", null, USER_OWNER1);
FieldChange fieldChange = new FieldChange().withName("description").withNewValue("newDescription");
FieldChange fieldChange = new FieldChange().withName("description").withOldValue("").withNewValue("newDescription");
ChangeDescription change =
getChangeDescription(entityInterface.getVersion()).withFieldsAdded(Collections.singletonList(fieldChange));
getChangeDescription(entityInterface.getVersion()).withFieldsUpdated(Collections.singletonList(fieldChange));
updateAndCheckEntity(request, OK, authHeaders(USER1.getEmail()), MINOR_UPDATE, change);
}
@ -829,6 +829,9 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
@Test
void put_entityNullDescriptionUpdate_200(TestInfo test) throws IOException {
if (!supportsEmptyDescription) {
return;
}
// Create entity with null description
K request = createRequest(getEntityName(test), null, "displayName", null);
T entity = createEntity(request, ADMIN_AUTH_HEADERS);
@ -1010,9 +1013,9 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
return;
}
// Create entity without description, owner
T entity = createEntity(createRequest(getEntityName(test), null, null, null), ADMIN_AUTH_HEADERS);
T entity = createEntity(createRequest(getEntityName(test), "", null, null), ADMIN_AUTH_HEADERS);
EntityInterface<T> entityInterface = getEntityInterface(entity);
assertListNull(entityInterface.getDescription(), entityInterface.getOwner());
assertListNull(entityInterface.getOwner());
entity = getEntity(entityInterface.getId(), ADMIN_AUTH_HEADERS);
entityInterface = getEntityInterface(entity);
@ -1029,7 +1032,9 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
// Field changes
ChangeDescription change = getChangeDescription(entityInterface.getVersion());
change.getFieldsAdded().add(new FieldChange().withName("description").withNewValue("description"));
change
.getFieldsUpdated()
.add(new FieldChange().withName("description").withOldValue("").withNewValue("description"));
if (supportsOwner) {
entityInterface.setOwner(TEAM_OWNER1);
change.getFieldsAdded().add(new FieldChange().withName(FIELD_OWNER).withNewValue(TEAM_OWNER1));
@ -1324,14 +1329,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
return createAndCheckEntity(create, authHeaders, create);
}
/**
* Helper function to create an entity, submit POST API request and validate response.
*
* @param create entity to be created
* @param authHeaders auth headers to be used for the PATCH API request
* @param created expected response from POST API after entity has been created
* @return entity response from the POST API
*/
/** Helper function to create an entity, submit POST API request and validate response. */
public final T createAndCheckEntity(K create, Map<String, String> authHeaders, K created) throws IOException {
// Validate an entity that is created has all the information set in create request
String updatedBy = TestUtils.getPrincipal(authHeaders);
@ -1429,17 +1427,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
return patchEntityAndCheck(updated, originalJson, authHeaders, updateType, expectedChange, updated);
}
/**
* Helper function to generate JSON PATCH, submit PATCH API request and validate response.
*
* @param updated entity to compare with response from PATCH API
* @param originalJson JSON representation of entity before the update
* @param authHeaders auth headers to be used for the PATCH API request
* @param updateType type of update, see {@link TestUtils.UpdateType}
* @param expectedChange change description that is expected from the PATCH API response
* @param update entity used to diff against originalJson to generate JSON PATCH for PATCH API test
* @return entity response from the PATCH API
*/
/** Helper function to generate JSON PATCH, submit PATCH API request and validate response. */
protected final T patchEntityAndCheck(
T updated,
String originalJson,

View File

@ -461,15 +461,14 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
//
// Patch operations on table1 created by POST operation. Columns can't be added or deleted. Only
// tags and
// description can be changed
// tags and description can be changed
//
String tableJson = JsonUtils.pojoToJson(table1);
c1 = table1.getColumns().get(0);
c1.withTags(singletonList(GLOSSARY1_TERM1_LABEL)); // c1 tag changed
c2 = table1.getColumns().get(1);
c2.withTags(Arrays.asList(USER_ADDRESS_TAG_LABEL, GLOSSARY1_TERM1_LABEL)); // c2 new tag added
c2.getTags().add(USER_ADDRESS_TAG_LABEL); // c2 new tag added
c2_a = c2.getChildren().get(0);
c2_a.withTags(singletonList(GLOSSARY1_TERM1_LABEL)); // c2.a tag changed
@ -1519,13 +1518,18 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
// Add a primary tag and derived tag both. The tag list must include derived tags only once.
String json = JsonUtils.pojoToJson(table);
table.getColumns().get(0).withTags(List.of(PERSONAL_DATA_TAG_LABEL, USER_ADDRESS_TAG_LABEL));
table.getColumns().get(0).withTags(List.of(GLOSSARY1_TERM1_LABEL, PERSONAL_DATA_TAG_LABEL, USER_ADDRESS_TAG_LABEL));
Table updatedTable = patchEntity(table.getId(), json, table, ADMIN_AUTH_HEADERS);
// Ensure only three tag labels are found - Manual tags PersonalData.Personal, User.Address
// and a derived tag PII.Sensitive
// Ensure only 4 tag labels are found - Manual tags PersonalData.Personal, User.Address, glossaryTerm1
// and a derived tag PII.Sensitive from glossary term1
List<TagLabel> updateTags = updatedTable.getColumns().get(0).getTags();
assertEquals(3, updateTags.size());
assertEquals(4, updateTags.size());
TagLabel glossaryTerm1 =
updateTags.stream().filter(t -> tagLabelMatch.test(t, GLOSSARY1_TERM1_LABEL)).findAny().orElse(null);
assertNotNull(glossaryTerm1);
assertEquals(LabelType.MANUAL, glossaryTerm1.getLabelType());
TagLabel userAddress =
updateTags.stream().filter(t -> tagLabelMatch.test(t, USER_ADDRESS_TAG_LABEL)).findAny().orElse(null);

View File

@ -18,6 +18,7 @@ package org.openmetadata.catalog.resources.glossary;
import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.catalog.util.TestUtils.assertListNull;
import static org.openmetadata.catalog.util.TestUtils.validateTagLabel;
import java.io.IOException;
import java.net.URISyntaxException;
@ -56,36 +57,40 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
@BeforeAll
public void setup(TestInfo test) throws IOException, URISyntaxException {
super.setup(test);
supportsEmptyDescription = false;
}
public void setupGlossaries() throws HttpResponseException {
public void setupGlossaries() throws IOException {
GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest();
CreateGlossary createGlossary = glossaryResourceTest.createRequest("g1", "", "", null);
GLOSSARY1 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
GLOSSARY1 = glossaryResourceTest.createAndCheckEntity(createGlossary, ADMIN_AUTH_HEADERS);
GLOSSARY1_REF = new GlossaryEntityInterface(GLOSSARY1).getEntityReference();
createGlossary = glossaryResourceTest.createRequest("g2", "", "", null);
GLOSSARY2 = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
GLOSSARY2 = glossaryResourceTest.createAndCheckEntity(createGlossary, ADMIN_AUTH_HEADERS);
GLOSSARY2_REF = new GlossaryEntityInterface(GLOSSARY2).getEntityReference();
GlossaryTermResourceTest glossaryTermResourceTest = new GlossaryTermResourceTest();
CreateGlossaryTerm createGlossaryTerm =
glossaryTermResourceTest
.createRequest("g1t1", null, "", null)
.createRequest("g1t1", "", "", null)
.withRelatedTerms(null)
.withGlossary(GLOSSARY1_REF);
GLOSSARY1_TERM1 = glossaryTermResourceTest.createEntity(createGlossaryTerm, ADMIN_AUTH_HEADERS);
.withGlossary(GLOSSARY1_REF)
.withTags(List.of(PII_SENSITIVE_TAG_LABEL, PERSONAL_DATA_TAG_LABEL));
GLOSSARY1_TERM1 = glossaryTermResourceTest.createAndCheckEntity(createGlossaryTerm, ADMIN_AUTH_HEADERS);
GLOSSARY1_TERM1_REF = new GlossaryTermEntityInterface(GLOSSARY1_TERM1).getEntityReference();
GLOSSARY1_TERM1_LABEL = getTagLabel(GLOSSARY1_TERM1);
validateTagLabel(GLOSSARY1_TERM1_LABEL);
createGlossaryTerm =
glossaryTermResourceTest
.createRequest("g2t1", null, "", null)
.withRelatedTerms(null)
.createRequest("g2t1", "", "", null)
.withRelatedTerms(List.of(GLOSSARY1_TERM1_REF))
.withGlossary(GLOSSARY2_REF);
GLOSSARY2_TERM1 = glossaryTermResourceTest.createEntity(createGlossaryTerm, ADMIN_AUTH_HEADERS);
GLOSSARY2_TERM1 = glossaryTermResourceTest.createAndCheckEntity(createGlossaryTerm, ADMIN_AUTH_HEADERS);
GLOSSARY2_TERM1_REF = new GlossaryTermEntityInterface(GLOSSARY2_TERM1).getEntityReference();
GLOSSARY2_TERM1_LABEL = getTagLabel(GLOSSARY2_TERM1);
validateTagLabel(GLOSSARY2_TERM1_LABEL);
}
private TagLabel getTagLabel(GlossaryTerm term) {

View File

@ -74,6 +74,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
"glossaryTerms",
GlossaryTermResource.FIELDS);
supportsAuthorizedMetadataOperations = false; // TODO why?
supportsEmptyDescription = false;
}
@Order(0)

View File

@ -377,7 +377,9 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline, CreatePip
Task taskEmptyDesc = new Task().withName("taskEmpty").withTaskUrl(new URI("http://localhost:0"));
tasks.add(taskEmptyDesc);
change.getFieldsAdded().add(new FieldChange().withName("tasks").withNewValue(tasks));
change.getFieldsAdded().add(new FieldChange().withName("description").withNewValue("newDescription"));
change
.getFieldsUpdated()
.add(new FieldChange().withName("description").withOldValue("").withNewValue("newDescription"));
// Create new request with all the Tasks
List<Task> updatedTasks = Stream.concat(TASKS.stream(), tasks.stream()).collect(Collectors.toList());

View File

@ -30,7 +30,6 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import javax.ws.rs.client.WebTarget;
@ -54,6 +53,7 @@ import org.openmetadata.catalog.type.CreateTagCategory.TagCategoryType;
import org.openmetadata.catalog.type.Tag;
import org.openmetadata.catalog.type.TagCategory;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.TagLabel.Source;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.FullyQualifiedName;
import org.openmetadata.catalog.util.JsonUtils;
@ -72,7 +72,7 @@ public class TagResourceTest extends CatalogApplicationTest {
new TagResourceTest().setupTags();
}
public void setupTags() throws HttpResponseException, JsonProcessingException {
public void setupTags() throws HttpResponseException {
TAGS_URL = "http://localhost:" + APP.getLocalPort() + "/api/v1/tags";
TagResourceTest tagResourceTest = new TagResourceTest();
EntityResourceTest.PERSONAL_DATA_TAG_LABEL = getTagLabel(FullyQualifiedName.add("PersonalData", "Personal"));
@ -100,7 +100,10 @@ public class TagResourceTest extends CatalogApplicationTest {
private TagLabel getTagLabel(String tagName) throws HttpResponseException {
Tag tag = TagResourceTest.getTag(tagName, ADMIN_AUTH_HEADERS);
return new TagLabel().withTagFQN(tag.getFullyQualifiedName()).withDescription(tag.getDescription());
return new TagLabel()
.withTagFQN(tag.getFullyQualifiedName())
.withDescription(tag.getDescription())
.withSource(Source.TAG);
}
@Test
@ -165,7 +168,7 @@ public class TagResourceTest extends CatalogApplicationTest {
}
@Test
void post_delete_validTagCategory_as_admin_201(TestInfo test) throws HttpResponseException, JsonProcessingException {
void post_delete_validTagCategory_as_admin_201(TestInfo test) throws HttpResponseException {
// POST .../tags/{newCategory} returns 201
String categoryName = test.getDisplayName().substring(0, 20); // Form a unique category name based on the test name
CreateTagCategory create =
@ -196,7 +199,7 @@ public class TagResourceTest extends CatalogApplicationTest {
}
@Test
void post_delete_validTags_as_admin_201(TestInfo test) throws HttpResponseException, JsonProcessingException {
void post_delete_validTags_as_admin_201(TestInfo test) throws HttpResponseException {
// Create tag category
String categoryName = test.getDisplayName().substring(0, 20);
CreateTagCategory create =
@ -279,7 +282,7 @@ public class TagResourceTest extends CatalogApplicationTest {
@Order(1)
@Test
void post_validTags_200() throws HttpResponseException, JsonProcessingException {
void post_validTags_200() throws HttpResponseException {
// POST .../tags/{category}/{primaryTag} to create primary tag
TagCategory category = getCategory(USER_TAG_CATEGORY.getName(), authHeaders("test@open-meatadata.org"));
CreateTag create = new CreateTag().withName("PrimaryTag").withDescription("description");
@ -341,7 +344,7 @@ public class TagResourceTest extends CatalogApplicationTest {
}
@Test
void put_tagCategory_200(TestInfo test) throws HttpResponseException {
void put_tagCategory_200(TestInfo test) {
// Update an existing tag category
String newCategoryName = test.getDisplayName().substring(0, 10);
CreateTagCategory create =
@ -379,7 +382,7 @@ public class TagResourceTest extends CatalogApplicationTest {
}
@Test
void put_primaryTag_200() throws HttpResponseException, JsonProcessingException {
void put_primaryTag_200() throws HttpResponseException {
// Update the tag name from User.Address to User.AddressUpdated
CreateTag create = new CreateTag().withName("AddressUpdated").withDescription("updatedDescription");
updatePrimaryTag(USER_TAG_CATEGORY.getName(), ADDRESS_TAG.getName(), create, ADMIN_AUTH_HEADERS);
@ -390,7 +393,7 @@ public class TagResourceTest extends CatalogApplicationTest {
}
@Test
void put_secondaryTag_200() throws HttpResponseException, JsonProcessingException {
void put_secondaryTag_200() throws HttpResponseException {
// Update the secondary tag name from User.PrimaryTag.SecondaryTag to User.PrimaryTag.SecondaryTag1
CreateTag create = new CreateTag().withName("SecondaryTag1").withDescription("description");
updateSecondaryTag(USER_TAG_CATEGORY.getName(), "PrimaryTag", "SecondaryTag", create, ADMIN_AUTH_HEADERS);
@ -455,13 +458,7 @@ public class TagResourceTest extends CatalogApplicationTest {
// Ensure POST returns the primary tag as expected
Tag returnedTag = TestUtils.post(target, create, Tag.class, authHeaders);
assertEquals(0.1, returnedTag.getVersion());
validate(
target.getUri().toString(),
returnedTag,
create.getName(),
create.getDescription(),
create.getAssociatedTags(),
updatedBy);
validate(target.getUri().toString(), returnedTag, create.getName(), create.getDescription(), updatedBy);
// Ensure GET returns the primary tag as expected
validate(
@ -469,26 +466,19 @@ public class TagResourceTest extends CatalogApplicationTest {
getTag(returnedTag.getFullyQualifiedName(), authHeaders),
create.getName(),
create.getDescription(),
create.getAssociatedTags(),
updatedBy);
return returnedTag;
}
private Tag createSecondaryTag(String category, String primaryTag, CreateTag create, Map<String, String> authHeaders)
throws HttpResponseException, JsonProcessingException {
throws HttpResponseException {
String updatedBy = TestUtils.getPrincipal(authHeaders);
WebTarget target = getResource("tags/" + category + "/" + primaryTag);
// Ensure POST returns the secondary tag as expected
Tag returnedTag = TestUtils.post(target, create, Tag.class, authHeaders);
assertEquals(0.1, returnedTag.getVersion());
validate(
target.getUri().toString(),
returnedTag,
create.getName(),
create.getDescription(),
create.getAssociatedTags(),
updatedBy);
validate(target.getUri().toString(), returnedTag, create.getName(), create.getDescription(), updatedBy);
// Ensure GET returns the primary tag as expected
validate(
@ -496,14 +486,12 @@ public class TagResourceTest extends CatalogApplicationTest {
getTag(returnedTag.getFullyQualifiedName(), authHeaders),
create.getName(),
create.getDescription(),
create.getAssociatedTags(),
updatedBy);
return returnedTag;
}
@SneakyThrows
private void updateCategory(String category, CreateTagCategory update, Map<String, String> authHeaders)
throws HttpResponseException {
private void updateCategory(String category, CreateTagCategory update, Map<String, String> authHeaders) {
String updatedBy = TestUtils.getPrincipal(authHeaders);
WebTarget target = getResource("tags/" + category);
@ -517,14 +505,14 @@ public class TagResourceTest extends CatalogApplicationTest {
}
private void updatePrimaryTag(String category, String primaryTag, CreateTag update, Map<String, String> authHeaders)
throws HttpResponseException, JsonProcessingException {
throws HttpResponseException {
String updatedBy = TestUtils.getPrincipal(authHeaders);
String parentHref = getResource("tags/" + category).getUri().toString();
WebTarget target = getResource("tags/" + category + "/" + primaryTag);
// Ensure PUT returns the updated primary tag
Tag returnedTag = TestUtils.put(target, update, Tag.class, Status.OK, authHeaders);
validate(parentHref, returnedTag, update.getName(), update.getDescription(), update.getAssociatedTags(), updatedBy);
validate(parentHref, returnedTag, update.getName(), update.getDescription(), updatedBy);
// Ensure GET returns the updated primary tag
validate(
@ -532,20 +520,19 @@ public class TagResourceTest extends CatalogApplicationTest {
getTag(returnedTag.getFullyQualifiedName(), authHeaders),
update.getName(),
update.getDescription(),
update.getAssociatedTags(),
updatedBy);
}
private void updateSecondaryTag(
String category, String primaryTag, String secondaryTag, CreateTag update, Map<String, String> authHeaders)
throws HttpResponseException, JsonProcessingException {
throws HttpResponseException {
String updatedBy = TestUtils.getPrincipal(authHeaders);
String parentHref = getResource("tags/" + category + "/" + primaryTag).getUri().toString();
WebTarget target = getResource("tags/" + category + "/" + primaryTag + "/" + secondaryTag);
// Ensure PUT returns the updated secondary tag
Tag returnedTag = TestUtils.put(target, update, Tag.class, Status.OK, authHeaders);
validate(parentHref, returnedTag, update.getName(), update.getDescription(), update.getAssociatedTags(), updatedBy);
validate(parentHref, returnedTag, update.getName(), update.getDescription(), updatedBy);
// Ensure GET returns the updated primary tag
validate(
@ -553,7 +540,6 @@ public class TagResourceTest extends CatalogApplicationTest {
getTag(returnedTag.getFullyQualifiedName(), authHeaders),
update.getName(),
update.getDescription(),
update.getAssociatedTags(),
updatedBy);
}
@ -617,20 +603,12 @@ public class TagResourceTest extends CatalogApplicationTest {
@SneakyThrows
private void validate(
String parentURI,
Tag actual,
String expectedName,
String expectedDescription,
List<String> expectedAssociatedTags,
String expectedUpdatedBy) {
String parentURI, Tag actual, String expectedName, String expectedDescription, String expectedUpdatedBy) {
LOG.info("Actual tag {}", JsonUtils.pojoToJson(actual));
validateHRef(parentURI, actual);
assertEquals(expectedName, actual.getName());
assertEquals(expectedDescription, actual.getDescription());
assertEquals(expectedUpdatedBy, actual.getUpdatedBy());
Collections.sort(expectedAssociatedTags);
Collections.sort(actual.getAssociatedTags());
assertEquals(expectedAssociatedTags, actual.getAssociatedTags());
}
/** Ensure the href in the children tags is correct */

View File

@ -116,11 +116,7 @@ public class RoleResourceTest extends EntityResourceTest<Role, CreateRole> {
return listEntities(Map.of("default", "true"), ADMIN_AUTH_HEADERS).getData();
}
/**
* Creates the given number of roles and sets one of them as the default role.
*
* @return the default role
*/
/** Creates the given number of roles and sets one of them as the default role. */
public Role createRolesAndSetDefault(TestInfo test, @Positive int numberOfRoles, @Positive int offset)
throws IOException {
// Create a set of roles.

View File

@ -184,9 +184,6 @@ class JwtFilterTest {
/**
* Creates the ContainerRequestsContext that is passed to the filter. This object can be quite complex, but the
* JwtFilter cares only about the Authorization header and request URI.
*
* @param jwt JWT in string format to be added to headers
* @return Mocked ContainerRequestContext with an Authorization header and request URI info
*/
private static ContainerRequestContext createRequestContextWithJwt(String jwt) {
MultivaluedHashMap<String, String> headers =

View File

@ -28,7 +28,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.json.JsonObject;
import javax.json.JsonPatch;
import javax.ws.rs.client.Entity;
@ -42,7 +41,9 @@ import org.eclipse.jetty.http.HttpStatus;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.function.Executable;
import org.openmetadata.catalog.api.services.DatabaseConnection;
import org.openmetadata.catalog.entity.data.GlossaryTerm;
import org.openmetadata.catalog.entity.teams.User;
import org.openmetadata.catalog.resources.glossary.GlossaryTermResourceTest;
import org.openmetadata.catalog.resources.tags.TagResourceTest;
import org.openmetadata.catalog.resources.teams.UserResourceTest;
import org.openmetadata.catalog.security.CatalogOpenIdAuthorizationRequestFilter;
@ -298,32 +299,46 @@ public final class TestUtils {
return;
}
actualList = listOrEmpty(actualList);
actualList.forEach(TestUtils::validateTagLabel);
// When tags from the expected list is added to an entity, the derived tags for those tags are automatically added
// So add to the expectedList, the derived tags before validating the tags
List<TagLabel> updatedExpectedList = new ArrayList<>(expectedList);
List<TagLabel> updatedExpectedList = new ArrayList<>();
EntityUtil.mergeTags(updatedExpectedList, expectedList);
for (TagLabel expected : expectedList) {
if (expected.getSource() != Source.TAG) {
continue; // TODO similar test for glossary
if (expected.getSource() == Source.GLOSSARY) {
GlossaryTerm glossaryTerm =
new GlossaryTermResourceTest().getEntityByName(expected.getTagFQN(), "tags", ADMIN_AUTH_HEADERS);
List<TagLabel> derived = new ArrayList<>();
for (TagLabel tag : listOrEmpty(glossaryTerm.getTags())) {
Tag associatedTag = TagResourceTest.getTag(tag.getTagFQN(), ADMIN_AUTH_HEADERS);
derived.add(
new TagLabel()
.withTagFQN(tag.getTagFQN())
.withState(expected.getState())
.withDescription(associatedTag.getDescription())
.withLabelType(TagLabel.LabelType.DERIVED));
}
EntityUtil.mergeTags(updatedExpectedList, derived);
}
Tag tag = TagResourceTest.getTag(expected.getTagFQN(), ADMIN_AUTH_HEADERS);
List<TagLabel> derived = new ArrayList<>();
for (String fqn : listOrEmpty(tag.getAssociatedTags())) {
Tag associatedTag = TagResourceTest.getTag(fqn, ADMIN_AUTH_HEADERS);
derived.add(
new TagLabel()
.withTagFQN(fqn)
.withState(expected.getState())
.withDescription(associatedTag.getDescription())
.withLabelType(TagLabel.LabelType.DERIVED));
}
updatedExpectedList.addAll(derived);
}
updatedExpectedList = updatedExpectedList.stream().distinct().collect(Collectors.toList());
updatedExpectedList.sort(EntityUtil.compareTagLabel);
actualList.sort(EntityUtil.compareTagLabel);
assertEquals(updatedExpectedList.size(), actualList.size());
assertEquals(updatedExpectedList, actualList);
}
public static void validateTagLabel(TagLabel label) {
assertNotNull(label.getTagFQN(), label.getTagFQN());
assertNotNull(label.getDescription(), label.getTagFQN());
assertNotNull(label.getLabelType(), label.getTagFQN());
assertNotNull(label.getSource(), label.getTagFQN());
assertNotNull(label.getState(), label.getTagFQN());
// TODO
// assertNotNull(label.getHref());
}
public static void checkUserFollowing(
UUID userId, UUID entityId, boolean expectedFollowing, Map<String, String> authHeaders)
throws HttpResponseException {
@ -402,9 +417,9 @@ public final class TestUtils {
}
}
public static void assertListNotEmpty(List... values) {
public static void assertListNotEmpty(List<?>... values) {
int index = 0;
for (List value : values) {
for (List<?> value : values) {
Assertions.assertFalse(value.isEmpty(), "List at index " + index + "is empty");
index++;
}

View File

@ -7,5 +7,5 @@ Provides metadata version information.
from incremental import Version
__version__ = Version("metadata", 0, 10, 0)
__version__ = Version("metadata", 0, 10, 0, dev=0)
__all__ = ["__version__"]