yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-24 05:58:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #3069 - Attempt 2 - Add PATCH support Glossary and GlossaryTerms (#3088)

Browse Source
This commit is contained in:
Suresh Srinivas 2022-03-02 18:22:09 -08:00 committed by GitHub
parent 71207de362
commit 6ad55972d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 318 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java
View File

@ -13,6 +13,8 @@
package org.openmetadata.catalog;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import java.io.IOException;
import java.net.URI;
@ -23,7 +25,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import javax.ws.rs.core.UriInfo;
import lombok.NonNull;
@ -163,7 +164,7 @@ public final class Entity {
}
public static void withHref(UriInfo uriInfo, List<EntityReference> list) {
Optional.ofNullable(list).orElse(Collections.emptyList()).forEach(ref -> withHref(uriInfo, ref));
listOrEmpty(list).forEach(ref -> withHref(uriInfo, ref));
}
public static EntityReference withHref(UriInfo uriInfo, EntityReference ref) {

7
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java
View File

@ -473,13 +473,6 @@ public interface CollectionDAO {
@Bind("relation") int relation,
@Bind("toEntity") String toEntity);
// Delete all the entity relationship fromID --- relation --> to any entity
@SqlUpdate(
"DELETE from entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity "
+ "AND relation = :relation")
int deleteFrom(
@Bind("fromId") String fromId, @Bind("fromEntity") String fromEntity, @Bind("relation") int relation);
// Delete all the entity relationship toId <-- relation -- entity of type fromEntity
@SqlUpdate(
"DELETE from entity_relationship WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation "

19
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java
View File

@ -16,15 +16,14 @@ package org.openmetadata.catalog.jdbi3;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.catalog.Entity.helper;
import static org.openmetadata.catalog.util.EntityUtil.toBoolean;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.net.URI;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
@ -205,10 +204,8 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
public void updateCharts(Dashboard original, Dashboard updated, EntityUpdater updater)
throws JsonProcessingException {
String dashboardId = updated.getId().toString();
// Remove all charts associated with this dashboard
daoCollection.relationshipDAO().deleteFrom(dashboardId, Entity.DASHBOARD, Relationship.HAS.ordinal(), Entity.CHART);
deleteFrom(updated.getId(), Entity.DASHBOARD, Relationship.HAS, Entity.CHART);
// Add relationship from dashboard to chart
if (updated.getCharts() != null) {
@ -386,18 +383,12 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
}
private void updateCharts() throws JsonProcessingException {
String dashboardId = updated.getId().toString();
// Remove all charts associated with this dashboard
daoCollection
.relationshipDAO()
.deleteFrom(dashboardId, Entity.DASHBOARD, Relationship.HAS.ordinal(), Entity.CHART);
deleteFrom(updated.getId(), Entity.DASHBOARD, Relationship.HAS, Entity.CHART);
// Add relationship from dashboard to chart
List<EntityReference> updatedCharts =
Optional.ofNullable(updated.getEntity().getCharts()).orElse(Collections.emptyList());
List<EntityReference> origCharts =
Optional.ofNullable(original.getEntity().getCharts()).orElse(Collections.emptyList());
List<EntityReference> updatedCharts = listOrEmpty(updated.getEntity().getCharts());
List<EntityReference> origCharts = listOrEmpty(original.getEntity().getCharts());
for (EntityReference chart : updatedCharts) {
addRelationship(updated.getId(), chart.getId(), Entity.DASHBOARD, Entity.CHART, Relationship.HAS);
}

8
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseRepository.java
View File

@ -66,9 +66,7 @@ public class DatabaseRepository extends EntityRepository<Database> {
@Transaction
public void deleteLocation(String databaseId) {
daoCollection
.relationshipDAO()
.deleteFrom(databaseId, Entity.DATABASE, Relationship.HAS.ordinal(), Entity.LOCATION);
deleteFrom(UUID.fromString(databaseId), Entity.DATABASE, Relationship.HAS, Entity.LOCATION);
}
@Override
@ -178,9 +176,7 @@ public class DatabaseRepository extends EntityRepository<Database> {
daoCollection.databaseDAO().findEntityById(databaseId);
daoCollection.locationDAO().findEntityById(locationId);
// A database has only one location.
daoCollection
.relationshipDAO()
.deleteFrom(databaseId.toString(), Entity.DATABASE, Relationship.HAS.ordinal(), Entity.LOCATION);
deleteFrom(databaseId, Entity.DATABASE, Relationship.HAS, Entity.LOCATION);
addRelationship(databaseId, locationId, Entity.DATABASE, Entity.LOCATION, Relationship.HAS);
return CREATED;
}

80
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java
View File

@ -22,6 +22,7 @@ import static org.openmetadata.catalog.util.EntityUtil.nextMajorVersion;
import static org.openmetadata.catalog.util.EntityUtil.nextVersion;
import static org.openmetadata.catalog.util.EntityUtil.objectMatch;
import static org.openmetadata.catalog.util.EntityUtil.toBoolean;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
@ -661,12 +662,7 @@ public abstract class EntityRepository<T> {
return new EntityHelper(entity);
}
/**
* Decorator class for Entity.
*
* @see Entity#helper(Object) to create a handler from an Entity
* @see Entity#r(EntityReference) to create a handler from an EntityReference
*/
/** Decorator class for Entity. */
public class EntityHelper {
private final Include isDeleted;
protected final EntityInterface<T> entityInterface;
@ -858,15 +854,20 @@ public abstract class EntityRepository<T> {
}
public int addRelationship(UUID fromId, UUID toId, String fromEntity, String toEntity, Relationship relationship) {
return daoCollection.relationshipDAO().insert(fromId, toId, fromEntity, toEntity, relationship.ordinal());
return addRelationship(fromId, toId, fromEntity, toEntity, relationship, false);
}
public int addBidirectionalRelationship(
UUID fromId, UUID toId, String fromEntity, String toEntity, Relationship relationship) {
if (fromId.compareTo(toId) < 0) {
return daoCollection.relationshipDAO().insert(fromId, toId, fromEntity, toEntity, relationship.ordinal());
public int addRelationship(
UUID fromId, UUID toId, String fromEntity, String toEntity, Relationship relationship, boolean bidirectional) {
UUID from = fromId;
UUID to = toId;
if (bidirectional && fromId.compareTo(toId) > 0) {
// For bidirectional relationship, instead of adding two row fromId -> toId and toId -> fromId, just add one
// row where fromId is alphabetically less than toId
from = toId;
to = fromId;
}
return daoCollection.relationshipDAO().insert(toId, fromId, toEntity, fromEntity, relationship.ordinal());
return daoCollection.relationshipDAO().insert(from, to, fromEntity, toEntity, relationship.ordinal());
}
public void setOwner(UUID ownedEntityId, String ownedEntityType, EntityReference owner) {
@ -879,6 +880,7 @@ public abstract class EntityRepository<T> {
public List<String> findBoth(
UUID entity1, String entityType1, Relationship relationship, String entity2, Boolean deleted) {
// Find bidirectional relationship
List<String> ids = new ArrayList<>();
ids.addAll(findFrom(entity1, entityType1, relationship, entity2, deleted));
ids.addAll(findTo(entity1, entityType1, relationship, entity2, deleted));
@ -886,17 +888,30 @@ public abstract class EntityRepository<T> {
}
public List<String> findFrom(
UUID toId, String toEntity, Relationship relationship, String fromEntity, Boolean deleted) {
return daoCollection
.relationshipDAO()
.findFrom(toId.toString(), toEntity, relationship.ordinal(), fromEntity, deleted);
UUID toId, String toEntityType, Relationship relationship, String fromEntityType, Boolean deleted) {
List<String> ret =
daoCollection
.relationshipDAO()
.findFrom(toId.toString(), toEntityType, relationship.ordinal(), fromEntityType, deleted);
return ret;
}
public List<String> findTo(
UUID fromId, String fromEntity, Relationship relationship, String toEntity, Boolean deleted) {
return daoCollection
.relationshipDAO()
.findTo(fromId.toString(), fromEntity, relationship.ordinal(), toEntity, deleted);
UUID fromId, String fromEntityType, Relationship relationship, String toEntityType, Boolean deleted) {
List<String> ret =
daoCollection
.relationshipDAO()
.findTo(fromId.toString(), fromEntityType, relationship.ordinal(), toEntityType, deleted);
return ret;
}
public void deleteTo(UUID toId, String toEntityType, Relationship relationship, String fromEntityType) {
daoCollection.relationshipDAO().deleteTo(toId.toString(), toEntityType, relationship.ordinal(), fromEntityType);
}
public void deleteFrom(UUID fromId, String fromEntityType, Relationship relationship, String toEntityType) {
// Remove relationships from original
daoCollection.relationshipDAO().deleteFrom(fromId.toString(), fromEntityType, relationship.ordinal(), toEntityType);
}
public void validateUsers(List<EntityReference> entityReferences) throws IOException {
@ -1043,7 +1058,7 @@ public abstract class EntityRepository<T> {
protected final void updateTags(String fqn, String fieldName, List<TagLabel> origTags, List<TagLabel> updatedTags)
throws IOException {
// Remove current entity tags in the database. It will be added back later from the merged tag list.
origTags = Optional.ofNullable(origTags).orElse(Collections.emptyList());
origTags = listOrEmpty(origTags);
// updatedTags cannot be immutable list, as we are adding the origTags to updatedTags even if its empty.
updatedTags = Optional.ofNullable(updatedTags).orElse(new ArrayList<>());
if (origTags.isEmpty() && updatedTags.isEmpty()) {
@ -1135,8 +1150,8 @@ public abstract class EntityRepository<T> {
List<K> deletedItems,
BiPredicate<K, K> typeMatch)
throws JsonProcessingException {
origList = Optional.ofNullable(origList).orElse(Collections.emptyList());
updatedList = Optional.ofNullable(updatedList).orElse(Collections.emptyList());
origList = listOrEmpty(origList);
updatedList = listOrEmpty(updatedList);
for (K stored : origList) {
// If an entry in the original list is not in updated list, then it is deleted during update
K u = updatedList.stream().filter(c -> typeMatch.test(c, stored)).findAny().orElse(null);
@ -1166,6 +1181,8 @@ public abstract class EntityRepository<T> {
/**
* Remove `fromEntityType:fromId` -- `relationType` ---> `toEntityType:origToRefs` Add `fromEntityType:fromId` --
* `relationType` ---> `toEntityType:updatedToRefs` and record it as change for entity field `field`.
*
* <p>When `bidirectional` is set to true, relationship from both side are replaced
*/
public final void updateToRelationships(
String field,
@ -1174,7 +1191,8 @@ public abstract class EntityRepository<T> {
Relationship relationshipType,
String toEntityType,
List<EntityReference> origToRefs,
List<EntityReference> updatedToRefs)
List<EntityReference> updatedToRefs,
boolean bidirectional)
throws JsonProcessingException {
List<EntityReference> added = new ArrayList<>();
List<EntityReference> deleted = new ArrayList<>();
@ -1183,12 +1201,13 @@ public abstract class EntityRepository<T> {
return;
}
// Remove relationships from original
daoCollection
.relationshipDAO()
.deleteFrom(fromId.toString(), fromEntityType, relationshipType.ordinal(), toEntityType);
deleteFrom(fromId, fromEntityType, relationshipType, toEntityType);
if (bidirectional) {
deleteTo(fromId, fromEntityType, relationshipType, toEntityType);
}
// Add relationships from updated
for (EntityReference ref : updatedToRefs) {
addRelationship(fromId, ref.getId(), fromEntityType, toEntityType, relationshipType);
addRelationship(fromId, ref.getId(), fromEntityType, toEntityType, relationshipType, bidirectional);
}
updatedToRefs.sort(EntityUtil.compareEntityReference);
origToRefs.sort(EntityUtil.compareEntityReference);
@ -1215,9 +1234,8 @@ public abstract class EntityRepository<T> {
return;
}
// Remove relationships from original
daoCollection
.relationshipDAO()
.deleteTo(toId.toString(), fromEntityType, relationshipType.ordinal(), toEntityType);
deleteTo(toId, toEntityType, relationshipType, fromEntityType);
// Add relationships from updated
for (EntityReference ref : updatedFromRefs) {
addRelationship(ref.getId(), toId, fromEntityType, toEntityType, relationshipType);

12
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java
View File

@ -18,14 +18,13 @@ package org.openmetadata.catalog.jdbi3;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.catalog.Entity.helper;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.net.URI;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
@ -102,7 +101,7 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
public void storeRelationships(Glossary glossary) {
setOwner(glossary, glossary.getOwner());
applyTags(glossary);
for (EntityReference reviewer : Optional.ofNullable(glossary.getReviewers()).orElse(Collections.emptyList())) {
for (EntityReference reviewer : listOrEmpty(glossary.getReviewers())) {
addRelationship(reviewer.getId(), glossary.getId(), Entity.USER, Entity.GLOSSARY, Relationship.REVIEWS);
}
}
@ -120,7 +119,6 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
private List<EntityReference> getReviewers(Glossary entity) throws IOException {
List<String> ids =
findFrom(entity.getId(), Entity.GLOSSARY, Relationship.REVIEWS, Entity.USER, entity.getDeleted());
System.out.println("XXX reviewers for " + entity.getId() + " " + ids);
return EntityUtil.populateEntityReferences(ids, Entity.USER);
}
@ -281,10 +279,8 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
}
private void updateReviewers(Glossary origGlossary, Glossary updatedGlossary) throws JsonProcessingException {
List<EntityReference> origUsers =
Optional.ofNullable(origGlossary.getReviewers()).orElse(Collections.emptyList());
List<EntityReference> updatedUsers =
Optional.ofNullable(updatedGlossary.getReviewers()).orElse(Collections.emptyList());
List<EntityReference> origUsers = listOrEmpty(origGlossary.getReviewers());
List<EntityReference> updatedUsers = listOrEmpty(updatedGlossary.getReviewers());
updateFromRelationships(
"reviewers",
Entity.USER,

70
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java
View File

@ -17,20 +17,20 @@
package org.openmetadata.catalog.jdbi3;
import static org.openmetadata.catalog.util.EntityUtil.stringMatch;
import static org.openmetadata.catalog.util.EntityUtil.termReferenceMatch;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.net.URI;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.TermReference;
import org.openmetadata.catalog.entity.data.GlossaryTerm;
import org.openmetadata.catalog.resources.glossary.GlossaryResource;
import org.openmetadata.catalog.resources.glossary.GlossaryTermResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
@ -43,8 +43,10 @@ import org.openmetadata.catalog.util.JsonUtils;
@Slf4j
public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
private static final Fields UPDATE_FIELDS = new Fields(GlossaryResource.ALLOWED_FIELDS, "tags,reviewers");
private static final Fields PATCH_FIELDS = new Fields(GlossaryResource.ALLOWED_FIELDS, "tags,reviewers");
private static final Fields UPDATE_FIELDS =
new Fields(GlossaryTermResource.ALLOWED_FIELDS, "tags,references,relatedTerms,reviewers,synonyms");
private static final Fields PATCH_FIELDS =
new Fields(GlossaryTermResource.ALLOWED_FIELDS, "tags,references,relatedTerms,reviewers,synonyms");
public GlossaryTermRepository(CollectionDAO dao) {
super(
@ -168,18 +170,24 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
Entity.GLOSSARY_TERM,
Relationship.PARENT_OF);
}
for (EntityReference relTerm : Optional.ofNullable(entity.getRelatedTerms()).orElse(Collections.emptyList())) {
for (EntityReference relTerm : listOrEmpty(entity.getRelatedTerms())) {
// Make this bidirectional relationship
addBidirectionalRelationship(
entity.getId(), relTerm.getId(), Entity.GLOSSARY_TERM, Entity.GLOSSARY_TERM, Relationship.RELATED_TO);
addRelationship(
entity.getId(), relTerm.getId(), Entity.GLOSSARY_TERM, Entity.GLOSSARY_TERM, Relationship.RELATED_TO, true);
}
for (EntityReference reviewer : Optional.ofNullable(entity.getReviewers()).orElse(Collections.emptyList())) {
for (EntityReference reviewer : listOrEmpty(entity.getReviewers())) {
addRelationship(reviewer.getId(), entity.getId(), Entity.USER, Entity.GLOSSARY_TERM, Relationship.REVIEWS);
}
applyTags(entity);
}
@Override
public void restorePatchAttributes(GlossaryTerm original, GlossaryTerm updated) {
// Patch can't update Children, Glossary, or Parent
updated.withChildren(original.getChildren()).withGlossary(original.getGlossary()).withParent(original.getParent());
}
protected EntityReference getGlossary(GlossaryTerm term) throws IOException {
// TODO check deleted
List<String> refs = findFrom(term.getId(), Entity.GLOSSARY_TERM, Relationship.CONTAINS, Entity.GLOSSARY, null);
@ -352,28 +360,58 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
@Override
public void entitySpecificUpdate() throws IOException {
updateStatus(original.getEntity(), updated.getEntity());
updateSynonyms(original.getEntity(), updated.getEntity());
updateReferences(original.getEntity(), updated.getEntity());
updateRelatedTerms(original.getEntity(), updated.getEntity());
updateReviewers(original.getEntity(), updated.getEntity());
}
private void updateStatus(GlossaryTerm origTerm, GlossaryTerm updatedTerm) throws JsonProcessingException {
// TODO Only list of allowed reviewers can change the status from DRAFT to APPROVED
recordChange("status", origTerm.getStatus(), updatedTerm.getStatus());
}
private void updateSynonyms(GlossaryTerm origTerm, GlossaryTerm updatedTerm) throws JsonProcessingException {
List<String> origSynonyms = Optional.ofNullable(origTerm.getSynonyms()).orElse(Collections.emptyList());
List<String> updatedSynonyms = Optional.ofNullable(updatedTerm.getSynonyms()).orElse(Collections.emptyList());
List<String> origSynonyms = listOrEmpty(origTerm.getSynonyms());
List<String> updatedSynonyms = listOrEmpty(updatedTerm.getSynonyms());
List<String> added = new ArrayList<>();
List<String> deleted = new ArrayList<>();
recordListChange("synonyms", origSynonyms, updatedSynonyms, added, deleted, stringMatch);
}
private void updateReferences(GlossaryTerm origTerm, GlossaryTerm updatedTerm) throws JsonProcessingException {
List<TermReference> origReferences = listOrEmpty(origTerm.getReferences());
List<TermReference> updatedReferences = listOrEmpty(updatedTerm.getReferences());
List<TermReference> added = new ArrayList<>();
List<TermReference> deleted = new ArrayList<>();
recordListChange("references", origReferences, updatedReferences, added, deleted, termReferenceMatch);
}
private void updateRelatedTerms(GlossaryTerm origTerm, GlossaryTerm updatedTerm) throws JsonProcessingException {
List<EntityReference> origRelated = listOrEmpty(origTerm.getRelatedTerms());
List<EntityReference> updatedRelated = listOrEmpty(updatedTerm.getRelatedTerms());
updateToRelationships(
"relatedTerms",
Entity.GLOSSARY_TERM,
origTerm.getId(),
Relationship.RELATED_TO,
Entity.GLOSSARY_TERM,
origRelated,
updatedRelated,
true);
}
private void updateReviewers(GlossaryTerm origTerm, GlossaryTerm updatedTerm) throws JsonProcessingException {
List<EntityReference> origUsers = Optional.ofNullable(origTerm.getReviewers()).orElse(Collections.emptyList());
List<EntityReference> updatedUsers =
Optional.ofNullable(updatedTerm.getReviewers()).orElse(Collections.emptyList());
List<EntityReference> origReviewers = listOrEmpty(origTerm.getReviewers());
List<EntityReference> updatedReviewers = listOrEmpty(updatedTerm.getReviewers());
updateFromRelationships(
"reviewers",
Entity.USER,
origUsers,
updatedUsers,
origReviewers,
updatedReviewers,
Relationship.REVIEWS,
Entity.GLOSSARY_TERM,
origTerm.getId());

7
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java
View File

@ -213,9 +213,7 @@ public class MlModelRepository extends EntityRepository<MlModel> {
}
public void removeDashboard(MlModel mlModel) {
daoCollection
.relationshipDAO()
.deleteTo(mlModel.getId().toString(), Entity.MLMODEL, Relationship.USES.ordinal(), Entity.DASHBOARD);
deleteTo(mlModel.getId(), Entity.MLMODEL, Relationship.USES, Entity.DASHBOARD);
}
public static class MlModelEntityInterface implements EntityInterface<MlModel> {
@ -444,9 +442,8 @@ public class MlModelRepository extends EntityRepository<MlModel> {
if (recordChange("dashboard", origDashboard, updatedDashboard, true, entityReferenceMatch)) {
// Remove the dashboard associated with the model, if any
String modelId = updatedModel.getId().toString();
if (origModel.getDashboard() != null) {
daoCollection.relationshipDAO().deleteTo(modelId, Entity.MLMODEL, Relationship.USES.ordinal(), "dashboard");
deleteTo(updatedModel.getId(), Entity.MLMODEL, Relationship.USES, Entity.DASHBOARD);
}
// Add relationship from model -- uses --> dashboard

7
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java
View File

@ -17,18 +17,17 @@ import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.catalog.Entity.PIPELINE_SERVICE;
import static org.openmetadata.catalog.Entity.helper;
import static org.openmetadata.catalog.util.EntityUtil.taskMatch;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.net.URI;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
@ -404,8 +403,8 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
// The API will only take care of marking tasks as added/updated/deleted based on the original
// and incoming changes.
List<Task> updatedTasks = Optional.ofNullable(updatedPipeline.getTasks()).orElse(Collections.emptyList());
List<Task> origTasks = Optional.ofNullable(origPipeline.getTasks()).orElse(Collections.emptyList());
List<Task> updatedTasks = listOrEmpty(updatedPipeline.getTasks());
List<Task> origTasks = listOrEmpty(origPipeline.getTasks());
List<Task> added = new ArrayList<>();
List<Task> deleted = new ArrayList<>();

18
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java
View File

@ -21,6 +21,7 @@ import static org.openmetadata.catalog.Entity.LOCATION;
import static org.openmetadata.catalog.Entity.TABLE;
import static org.openmetadata.catalog.Entity.helper;
import static org.openmetadata.catalog.util.EntityUtil.getColumnField;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.common.utils.CommonUtil.parseDate;
import com.fasterxml.jackson.core.JsonProcessingException;
@ -38,7 +39,6 @@ import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.UUID;
import java.util.function.BiPredicate;
import java.util.function.Function;
@ -221,7 +221,7 @@ public class TableRepository extends EntityRepository<Table> {
Table table = daoCollection.tableDAO().findEntityById(tableId);
EntityReference location = daoCollection.locationDAO().findEntityReferenceById(locationId);
// A table has only one location.
daoCollection.relationshipDAO().deleteFrom(tableId.toString(), TABLE, Relationship.HAS.ordinal(), LOCATION);
deleteFrom(tableId, TABLE, Relationship.HAS, LOCATION);
addRelationship(tableId, locationId, TABLE, LOCATION, Relationship.HAS);
setFields(table, Fields.EMPTY_FIELDS);
return table.withLocation(location);
@ -409,7 +409,7 @@ public class TableRepository extends EntityRepository<Table> {
table.setDescription(dataModel.getDescription());
}
// Carry forward the column description from the model to table columns, if empty
for (Column modelColumn : Optional.ofNullable(dataModel.getColumns()).orElse(Collections.emptyList())) {
for (Column modelColumn : listOrEmpty(dataModel.getColumns())) {
Column stored =
table.getColumns().stream()
.filter(c -> EntityUtil.columnNameMatch.test(c, modelColumn))
@ -429,7 +429,7 @@ public class TableRepository extends EntityRepository<Table> {
@Transaction
public void deleteLocation(String tableId) {
daoCollection.relationshipDAO().deleteFrom(tableId, TABLE, Relationship.HAS.ordinal(), LOCATION);
deleteFrom(UUID.fromString(tableId), TABLE, Relationship.HAS, LOCATION);
}
private void setColumnFQN(String parentFQN, List<Column> columns) {
@ -579,7 +579,7 @@ public class TableRepository extends EntityRepository<Table> {
}
private void getColumnTags(boolean setTags, List<Column> columns) {
for (Column c : Optional.ofNullable(columns).orElse(Collections.emptyList())) {
for (Column c : listOrEmpty(columns)) {
c.setTags(setTags ? getTags(c.getFullyQualifiedName()) : null);
getColumnTags(setTags, c.getChildren());
}
@ -809,7 +809,7 @@ public class TableRepository extends EntityRepository<Table> {
private void getColumnTests(boolean setTests, Table table) throws IOException {
List<Column> columns = table.getColumns();
for (Column c : Optional.ofNullable(columns).orElse(Collections.emptyList())) {
for (Column c : listOrEmpty(columns)) {
c.setColumnTests(setTests ? getColumnTests(table, c.getName()) : null);
}
}
@ -980,10 +980,8 @@ public class TableRepository extends EntityRepository<Table> {
}
private void updateConstraints(Table origTable, Table updatedTable) throws JsonProcessingException {
List<TableConstraint> origConstraints =
Optional.ofNullable(origTable.getTableConstraints()).orElse(Collections.emptyList());
List<TableConstraint> updatedConstraints =
Optional.ofNullable(updatedTable.getTableConstraints()).orElse(Collections.emptyList());
List<TableConstraint> origConstraints = listOrEmpty(origTable.getTableConstraints());
List<TableConstraint> updatedConstraints = listOrEmpty(updatedTable.getTableConstraints());
origConstraints.sort(EntityUtil.compareTableConstraint);
origConstraints.stream().map(TableConstraint::getColumns).forEach(Collections::sort);

8
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TagRepository.java
View File

@ -13,14 +13,14 @@
package org.openmetadata.catalog.jdbi3;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.mapper.RowMapper;
import org.jdbi.v3.sqlobject.transaction.Transaction;
@ -201,7 +201,7 @@ public class TagRepository {
LOG.info("Added tag {}", tag.getFullyQualifiedName());
// Then add the children
for (Tag children : Optional.ofNullable(tags).orElse(Collections.emptyList())) {
for (Tag children : listOrEmpty(tags)) {
children.setChildren(null); // No children allowed for the leaf tag
children.setFullyQualifiedName(children.getFullyQualifiedName() + "." + children.getName());
LOG.info("Added tag {}", children.getFullyQualifiedName());
@ -229,7 +229,7 @@ public class TagRepository {
// Get tags under the given tag
List<Tag> tagList = new ArrayList<>();
for (String json : Optional.ofNullable(tagJsons).orElse(Collections.emptyList())) {
for (String json : listOrEmpty(tagJsons)) {
Tag childTag = setFields(JsonUtils.readValue(json, Tag.class), fields);
tagList.add(populateChildrenTags(childTag, fields));
}

22
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java
View File

@ -14,14 +14,13 @@
package org.openmetadata.catalog.jdbi3;
import static org.openmetadata.catalog.util.EntityUtil.toBoolean;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.teams.Team;
@ -107,10 +106,10 @@ public class TeamRepository extends EntityRepository<Team> {
@Override
public void storeRelationships(Team team) {
for (EntityReference user : Optional.ofNullable(team.getUsers()).orElse(Collections.emptyList())) {
for (EntityReference user : listOrEmpty(team.getUsers())) {
addRelationship(team.getId(), user.getId(), Entity.TEAM, Entity.USER, Relationship.HAS);
}
for (EntityReference defaultRole : Optional.ofNullable(team.getDefaultRoles()).orElse(Collections.emptyList())) {
for (EntityReference defaultRole : listOrEmpty(team.getDefaultRoles())) {
addRelationship(team.getId(), defaultRole.getId(), Entity.TEAM, Entity.ROLE, Relationship.HAS);
}
}
@ -274,17 +273,15 @@ public class TeamRepository extends EntityRepository<Team> {
}
private void updateUsers(Team origTeam, Team updatedTeam) throws JsonProcessingException {
List<EntityReference> origUsers = Optional.ofNullable(origTeam.getUsers()).orElse(Collections.emptyList());
List<EntityReference> updatedUsers = Optional.ofNullable(updatedTeam.getUsers()).orElse(Collections.emptyList());
List<EntityReference> origUsers = listOrEmpty(origTeam.getUsers());
List<EntityReference> updatedUsers = listOrEmpty(updatedTeam.getUsers());
updateToRelationships(
"users", Entity.TEAM, origTeam.getId(), Relationship.HAS, Entity.USER, origUsers, updatedUsers);
"users", Entity.TEAM, origTeam.getId(), Relationship.HAS, Entity.USER, origUsers, updatedUsers, false);
}
private void updateDefaultRoles(Team origTeam, Team updatedTeam) throws JsonProcessingException {
List<EntityReference> origDefaultRoles =
Optional.ofNullable(origTeam.getDefaultRoles()).orElse(Collections.emptyList());
List<EntityReference> updatedDefaultRoles =
Optional.ofNullable(updatedTeam.getDefaultRoles()).orElse(Collections.emptyList());
List<EntityReference> origDefaultRoles = listOrEmpty(origTeam.getDefaultRoles());
List<EntityReference> updatedDefaultRoles = listOrEmpty(updatedTeam.getDefaultRoles());
updateToRelationships(
"defaultRoles",
Entity.TEAM,
@ -292,7 +289,8 @@ public class TeamRepository extends EntityRepository<Team> {
Relationship.HAS,
Entity.ROLE,
origDefaultRoles,
updatedDefaultRoles);
updatedDefaultRoles,
false);
}
}
}

29
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java
View File

@ -15,6 +15,7 @@ package org.openmetadata.catalog.jdbi3;
import static org.openmetadata.catalog.Entity.helper;
import static org.openmetadata.catalog.util.EntityUtil.toBoolean;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
@ -23,7 +24,6 @@ import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
@ -68,10 +68,7 @@ public class UserRepository extends EntityRepository<User> {
@Override
public void prepare(User user) throws IOException, ParseException {
// Get roles assigned to the user.
Set<UUID> roleIds =
Optional.ofNullable(user.getRoles()).orElse(Collections.emptyList()).stream()
.map(EntityReference::getId)
.collect(Collectors.toSet());
Set<UUID> roleIds = listOrEmpty(user.getRoles()).stream().map(EntityReference::getId).collect(Collectors.toSet());
// Get default role set up globally.
daoCollection.roleDAO().getDefaultRolesIds().forEach(roleIdStr -> roleIds.add(UUID.fromString(roleIdStr)));
// Get default roles from the teams that the user belongs to.
@ -87,7 +84,7 @@ public class UserRepository extends EntityRepository<User> {
}
private List<EntityReference> getTeamDefaultRoles(User user) throws IOException, ParseException {
List<EntityReference> teamsRef = Optional.ofNullable(user.getTeams()).orElse(Collections.emptyList());
List<EntityReference> teamsRef = listOrEmpty(user.getTeams());
List<EntityReference> defaultRoles = new ArrayList<>();
for (EntityReference teamRef : teamsRef) {
Team team = Entity.getEntity(teamRef, new Fields(List.of("defaultRoles")));
@ -208,14 +205,14 @@ public class UserRepository extends EntityRepository<User> {
}
private void assignRoles(User user, List<EntityReference> roles) {
roles = Optional.ofNullable(roles).orElse(Collections.emptyList());
roles = listOrEmpty(roles);
for (EntityReference role : roles) {
addRelationship(user.getId(), role.getId(), Entity.USER, Entity.ROLE, Relationship.HAS);
}
}
private void assignTeams(User user, List<EntityReference> teams) {
teams = Optional.ofNullable(teams).orElse(Collections.emptyList());
teams = listOrEmpty(teams);
for (EntityReference team : teams) {
addRelationship(team.getId(), user.getId(), Entity.TEAM, Entity.USER, Relationship.HAS);
}
@ -361,13 +358,11 @@ public class UserRepository extends EntityRepository<User> {
private void updateRoles(User origUser, User updatedUser) throws IOException {
// Remove roles from original and add roles from updated
daoCollection
.relationshipDAO()
.deleteFrom(origUser.getId().toString(), Entity.USER, Relationship.HAS.ordinal(), Entity.ROLE);
deleteFrom(origUser.getId(), Entity.USER, Relationship.HAS, Entity.ROLE);
assignRoles(updatedUser, updatedUser.getRoles());
List<EntityReference> origRoles = Optional.ofNullable(origUser.getRoles()).orElse(Collections.emptyList());
List<EntityReference> updatedRoles = Optional.ofNullable(updatedUser.getRoles()).orElse(Collections.emptyList());
List<EntityReference> origRoles = listOrEmpty(origUser.getRoles());
List<EntityReference> updatedRoles = listOrEmpty(updatedUser.getRoles());
origRoles.sort(EntityUtil.compareEntityReference);
updatedRoles.sort(EntityUtil.compareEntityReference);
@ -379,13 +374,11 @@ public class UserRepository extends EntityRepository<User> {
private void updateTeams(User origUser, User updatedUser) throws JsonProcessingException {
// Remove teams from original and add teams from updated
daoCollection
.relationshipDAO()
.deleteTo(origUser.getId().toString(), Entity.USER, Relationship.HAS.ordinal(), Entity.TEAM);
deleteTo(origUser.getId(), Entity.USER, Relationship.HAS, Entity.TEAM);
assignTeams(updatedUser, updatedUser.getTeams());
List<EntityReference> origTeams = Optional.ofNullable(origUser.getTeams()).orElse(Collections.emptyList());
List<EntityReference> updatedTeams = Optional.ofNullable(updatedUser.getTeams()).orElse(Collections.emptyList());
List<EntityReference> origTeams = listOrEmpty(origUser.getTeams());
List<EntityReference> updatedTeams = listOrEmpty(updatedUser.getTeams());
origTeams.sort(EntityUtil.compareEntityReference);
updatedTeams.sort(EntityUtil.compareEntityReference);

6
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java
View File

@ -13,6 +13,8 @@
package org.openmetadata.catalog.resources.charts;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
@ -26,9 +28,7 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
@ -78,7 +78,7 @@ public class ChartResource {
private final Authorizer authorizer;
public static ResultList<Chart> addHref(UriInfo uriInfo, ResultList<Chart> charts) {
Optional.ofNullable(charts.getData()).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(charts.getData()).forEach(i -> addHref(uriInfo, i));
return charts;
}

6
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java
View File

@ -13,6 +13,8 @@
package org.openmetadata.catalog.resources.dashboards;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
@ -26,10 +28,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
@ -79,7 +79,7 @@ public class DashboardResource {
private final Authorizer authorizer;
public static ResultList<Dashboard> addHref(UriInfo uriInfo, ResultList<Dashboard> dashboards) {
Optional.ofNullable(dashboards.getData()).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(dashboards.getData()).forEach(i -> addHref(uriInfo, i));
return dashboards;
}

6
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java
View File

@ -13,6 +13,8 @@
package org.openmetadata.catalog.resources.glossary;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.google.inject.Inject;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
@ -27,10 +29,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
@ -81,7 +81,7 @@ public class GlossaryResource {
private final Authorizer authorizer;
public static List<Glossary> addHref(UriInfo uriInfo, List<Glossary> glossaries) {
Optional.ofNullable(glossaries).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(glossaries).forEach(i -> addHref(uriInfo, i));
return glossaries;
}

8
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResource.java
View File

@ -13,6 +13,8 @@
package org.openmetadata.catalog.resources.glossary;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.google.inject.Inject;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
@ -27,10 +29,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
@ -83,7 +83,7 @@ public class GlossaryTermResource {
private final Authorizer authorizer;
public static List<GlossaryTerm> addHref(UriInfo uriInfo, List<GlossaryTerm> terms) {
Optional.ofNullable(terms).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(terms).forEach(i -> addHref(uriInfo, i));
return terms;
}
@ -117,7 +117,7 @@ public class GlossaryTermResource {
}
static final String FIELDS = "children,relatedTerms,reviewers,tags";
protected static final List<String> ALLOWED_FIELDS = Entity.getEntityFields(GlossaryTerm.class);
public static final List<String> ALLOWED_FIELDS = Entity.getEntityFields(GlossaryTerm.class);
@GET
@Valid

7
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/AirflowPipelineResource.java
View File

@ -14,6 +14,7 @@
package org.openmetadata.catalog.resources.operations;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
@ -28,10 +29,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
@ -87,7 +86,7 @@ public class AirflowPipelineResource {
private CatalogApplicationConfig config;
public static ResultList<AirflowPipeline> addHref(UriInfo uriInfo, ResultList<AirflowPipeline> ingestions) {
Optional.ofNullable(ingestions.getData()).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(ingestions.getData()).forEach(i -> addHref(uriInfo, i));
return ingestions;
}
@ -465,7 +464,7 @@ public class AirflowPipelineResource {
}
public void addStatus(List<AirflowPipeline> airflowPipelines) {
Optional.ofNullable(airflowPipelines).orElse(Collections.emptyList()).forEach(this::addStatus);
listOrEmpty(airflowPipelines).forEach(this::addStatus);
}
private AirflowPipeline addStatus(AirflowPipeline airflowPipeline) {

6
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/pipelines/PipelineResource.java
View File

@ -13,6 +13,8 @@
package org.openmetadata.catalog.resources.pipelines;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
@ -26,10 +28,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
@ -80,7 +80,7 @@ public class PipelineResource {
private final Authorizer authorizer;
public static ResultList<Pipeline> addHref(UriInfo uriInfo, ResultList<Pipeline> pipelines) {
Optional.ofNullable(pipelines.getData()).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(pipelines.getData()).forEach(i -> addHref(uriInfo, i));
return pipelines;
}

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/policies/PolicyResource.java
View File

@ -14,6 +14,7 @@
package org.openmetadata.catalog.resources.policies;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
@ -28,10 +29,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
@ -84,7 +83,7 @@ public class PolicyResource {
private final Authorizer authorizer;
public static ResultList<Policy> addHref(UriInfo uriInfo, ResultList<Policy> policies) {
Optional.ofNullable(policies.getData()).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(policies.getData()).forEach(i -> addHref(uriInfo, i));
return policies;
}

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/dashboard/DashboardServiceResource.java
View File

@ -14,6 +14,7 @@
package org.openmetadata.catalog.resources.services.dashboard;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
@ -25,10 +26,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.validation.Valid;
import javax.validation.constraints.Max;
@ -78,7 +77,7 @@ public class DashboardServiceResource {
public static final List<String> ALLOWED_FIELDS = Entity.getEntityFields(DashboardService.class);
public static ResultList<DashboardService> addHref(UriInfo uriInfo, ResultList<DashboardService> services) {
Optional.ofNullable(services.getData()).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(services.getData()).forEach(i -> addHref(uriInfo, i));
return services;
}

3
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/database/DatabaseServiceResource.java
View File

@ -14,6 +14,7 @@
package org.openmetadata.catalog.resources.services.database;
import static org.openmetadata.catalog.fernet.Fernet.isTokenized;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
@ -85,7 +86,7 @@ public class DatabaseServiceResource {
private final Fernet fernet;
public static ResultList<DatabaseService> addHref(UriInfo uriInfo, ResultList<DatabaseService> dbServices) {
Optional.ofNullable(dbServices.getData()).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(dbServices.getData()).forEach(i -> addHref(uriInfo, i));
return dbServices;
}

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/messaging/MessagingServiceResource.java
View File

@ -14,6 +14,7 @@
package org.openmetadata.catalog.resources.services.messaging;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
@ -25,10 +26,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.validation.Valid;
import javax.validation.constraints.Max;
@ -78,7 +77,7 @@ public class MessagingServiceResource {
public static final List<String> ALLOWED_FIELDS = Entity.getEntityFields(MessagingService.class);
public static ResultList<MessagingService> addHref(UriInfo uriInfo, ResultList<MessagingService> services) {
Optional.ofNullable(services.getData()).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(services.getData()).forEach(i -> addHref(uriInfo, i));
return services;
}

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/pipeline/PipelineServiceResource.java
View File

@ -14,6 +14,7 @@
package org.openmetadata.catalog.resources.services.pipeline;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
@ -25,10 +26,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.validation.Valid;
import javax.validation.constraints.Max;
@ -78,7 +77,7 @@ public class PipelineServiceResource {
public static final List<String> ALLOWED_FIELDS = Entity.getEntityFields(PipelineService.class);
public static ResultList<PipelineService> addHref(UriInfo uriInfo, ResultList<PipelineService> services) {
Optional.ofNullable(services.getData()).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(services.getData()).forEach(i -> addHref(uriInfo, i));
return services;
}

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/storage/StorageServiceResource.java
View File

@ -14,6 +14,7 @@
package org.openmetadata.catalog.resources.services.storage;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
@ -25,10 +26,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.validation.Valid;
import javax.validation.constraints.Max;
@ -78,7 +77,7 @@ public class StorageServiceResource {
public static final List<String> ALLOWED_FIELDS = Entity.getEntityFields(StorageService.class);
public static ResultList<StorageService> addHref(UriInfo uriInfo, ResultList<StorageService> services) {
Optional.ofNullable(services.getData()).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(services.getData()).forEach(i -> addHref(uriInfo, i));
return services;
}

6
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/tags/TagResource.java
View File

@ -13,6 +13,8 @@
package org.openmetadata.catalog.resources.tags;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.v3.oas.annotations.Operation;
@ -22,10 +24,8 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.io.IOException;
import java.net.URI;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Pattern;
import javax.validation.Valid;
import javax.ws.rs.GET;
@ -473,7 +473,7 @@ public class TagResource {
}
private void addHref(URI parentHref, List<Tag> tags) {
for (Tag tag : Optional.ofNullable(tags).orElse(Collections.emptyList())) {
for (Tag tag : listOrEmpty(tags)) {
addHref(parentHref, tag);
}
}

6
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/UserResource.java
View File

@ -13,6 +13,8 @@
package org.openmetadata.catalog.resources.teams;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import io.dropwizard.jersey.PATCH;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
@ -27,10 +29,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.json.JsonObject;
import javax.json.JsonPatch;
@ -163,7 +163,7 @@ public class UserResource {
} else { // Forward paging or first page
users = dao.listAfter(uriInfo, fields, teamParam, limitParam, after, include);
}
Optional.ofNullable(users.getData()).orElse(Collections.emptyList()).forEach(u -> addHref(uriInfo, u));
listOrEmpty(users.getData()).forEach(u -> addHref(uriInfo, u));
return users;
}

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java
View File

@ -14,6 +14,7 @@
package org.openmetadata.catalog.resources.topics;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.google.inject.Inject;
import io.swagger.annotations.Api;
@ -29,9 +30,7 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
@ -81,7 +80,7 @@ public class TopicResource {
private final Authorizer authorizer;
public static ResultList<Topic> addHref(UriInfo uriInfo, ResultList<Topic> topics) {
Optional.ofNullable(topics.getData()).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
listOrEmpty(topics.getData()).forEach(i -> addHref(uriInfo, i));
return topics;
}

12
catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java
View File

@ -15,6 +15,7 @@ package org.openmetadata.catalog.util;
import static org.openmetadata.catalog.type.Include.ALL;
import static org.openmetadata.catalog.type.Include.DELETED;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import java.io.IOException;
import java.util.ArrayList;
@ -35,6 +36,7 @@ import lombok.extern.slf4j.Slf4j;
import org.joda.time.Period;
import org.joda.time.format.ISOPeriodFormat;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.TermReference;
import org.openmetadata.catalog.entity.data.GlossaryTerm;
import org.openmetadata.catalog.entity.teams.Team;
import org.openmetadata.catalog.entity.teams.User;
@ -128,6 +130,9 @@ public final class EntityUtil {
public static final BiPredicate<GlossaryTerm, GlossaryTerm> glossaryTermMatch =
(filter1, filter2) -> filter1.getFullyQualifiedName().equals(filter2.getFullyQualifiedName());
public static final BiPredicate<TermReference, TermReference> termReferenceMatch =
(ref1, ref2) -> ref1.getName().equals(ref2.getName()) && ref1.getEndpoint().equals(ref2.getEndpoint());
private EntityUtil() {}
/** Validate Ingestion Schedule */
@ -276,7 +281,7 @@ public final class EntityUtil {
/** Apply tags {@code tagLabels} to the entity or field identified by {@code targetFQN} */
public static void applyTags(TagDAO tagDAO, List<TagLabel> tagLabels, String targetFQN) {
for (TagLabel tagLabel : Optional.ofNullable(tagLabels).orElse(Collections.emptyList())) {
for (TagLabel tagLabel : listOrEmpty(tagLabels)) {
String json = tagDAO.findTag(tagLabel.getTagFQN());
if (json == null) {
// Invalid TagLabel
@ -292,7 +297,7 @@ public final class EntityUtil {
public static List<TagLabel> getDerivedTags(TagDAO tagDAO, TagLabel tagLabel, Tag tag) throws IOException {
List<TagLabel> derivedTags = new ArrayList<>();
for (String fqn : Optional.ofNullable(tag.getAssociatedTags()).orElse(Collections.emptyList())) {
for (String fqn : listOrEmpty(tag.getAssociatedTags())) {
String json = tagDAO.findTag(fqn);
if (json == null) {
// Invalid TagLabel
@ -507,7 +512,8 @@ public final class EntityUtil {
escapeReservedChars(column.getChildren());
} else if (object instanceof TableConstraint) {
TableConstraint constraint = (TableConstraint) object;
constraint.setColumns(constraint.getColumns().stream().map(s -> replaceDot(s)).collect(Collectors.toList()));
constraint.setColumns(
constraint.getColumns().stream().map(EntityUtil::replaceDot).collect(Collectors.toList()));
} else if (object instanceof Task) {
Task task = (Task) object;
task.setDisplayName(task.getDisplayName() != null ? task.getDisplayName() : task.getName());

9
catalog-rest-service/src/main/resources/json/schema/api/data/createGlossaryTerm.json
View File

@ -32,17 +32,16 @@
"$ref": "../../entity/data/glossaryTerm.json#/definitions/name"
}
},
"children": {
"description": "Other glossary terms that are children of this glossary term.",
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList"
},
"relatedTerms": {
"description": "Other glossary terms that are related to this glossary term.",
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList"
},
"references": {
"description": "Link to a reference from an external glossary.",
"$ref": "../../entity/data/glossaryTerm.json#/definitions/termReference"
"type": "array",
"items": {
"$ref": "../../entity/data/glossaryTerm.json#/definitions/termReference"
}
},
"reviewers": {
"description": "User names of the reviewers for this glossary.",

5
catalog-rest-service/src/main/resources/json/schema/entity/data/glossaryTerm.json
View File

@ -80,7 +80,10 @@
},
"references": {
"description": "Link to a reference from an external glossary.",
"$ref": "#/definitions/termReference"
"type": "array",
"items": {
"$ref": "../../entity/data/glossaryTerm.json#/definitions/termReference"
}
},
"version": {
"description": "Metadata version of the entity.",

43
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java
View File

@ -80,6 +80,7 @@ import org.openmetadata.catalog.CatalogApplicationTest;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateChart;
import org.openmetadata.catalog.api.data.CreateDatabase;
import org.openmetadata.catalog.api.data.TermReference;
import org.openmetadata.catalog.api.services.CreateDashboardService;
import org.openmetadata.catalog.api.services.CreateDashboardService.DashboardServiceType;
import org.openmetadata.catalog.api.services.CreateDatabaseService;
@ -105,6 +106,10 @@ import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface
import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.PipelineServiceRepository.PipelineServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.RoleRepository.RoleEntityInterface;
import org.openmetadata.catalog.jdbi3.StorageServiceRepository.StorageServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.TeamRepository.TeamEntityInterface;
import org.openmetadata.catalog.jdbi3.UserRepository.UserEntityInterface;
import org.openmetadata.catalog.resources.charts.ChartResourceTest;
import org.openmetadata.catalog.resources.databases.DatabaseResourceTest;
import org.openmetadata.catalog.resources.events.EventResource.ChangeEventList;
@ -160,6 +165,8 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
public static User USER1;
public static EntityReference USER_OWNER1;
public static User USER2;
public static EntityReference USER_OWNER2;
public static Team TEAM1;
public static EntityReference TEAM_OWNER1;
public static User USER_WITH_DATA_STEWARD_ROLE;
@ -242,12 +249,15 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
UserResourceTest userResourceTest = new UserResourceTest();
USER1 = userResourceTest.createEntity(userResourceTest.createRequest(test), ADMIN_AUTH_HEADERS);
USER_OWNER1 = new EntityReference().withId(USER1.getId()).withType(Entity.USER);
USER_OWNER1 = new UserEntityInterface(USER1).getEntityReference();
USER2 = userResourceTest.createEntity(userResourceTest.createRequest(test, 1), ADMIN_AUTH_HEADERS);
USER_OWNER2 = new UserEntityInterface(USER2).getEntityReference();
RoleResourceTest roleResourceTest = new RoleResourceTest();
DATA_STEWARD_ROLE =
roleResourceTest.getEntityByName(DATA_STEWARD_ROLE_NAME, RoleResource.FIELDS, ADMIN_AUTH_HEADERS);
DATA_STEWARD_ROLE_REFERENCE = new EntityReference().withId(DATA_STEWARD_ROLE.getId()).withType(Entity.ROLE);
DATA_STEWARD_ROLE_REFERENCE = new RoleEntityInterface(DATA_STEWARD_ROLE).getEntityReference();
USER_WITH_DATA_STEWARD_ROLE =
userResourceTest.createEntity(
userResourceTest
@ -256,7 +266,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
ADMIN_AUTH_HEADERS);
DATA_CONSUMER_ROLE =
roleResourceTest.getEntityByName(DATA_CONSUMER_ROLE_NAME, RoleResource.FIELDS, ADMIN_AUTH_HEADERS);
DATA_CONSUMER_ROLE_REFERENCE = new EntityReference().withId(DATA_CONSUMER_ROLE.getId()).withType(Entity.ROLE);
DATA_CONSUMER_ROLE_REFERENCE = new RoleEntityInterface(DATA_CONSUMER_ROLE).getEntityReference();
USER_WITH_DATA_CONSUMER_ROLE =
userResourceTest.createEntity(
userResourceTest
@ -266,10 +276,10 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
TeamResourceTest teamResourceTest = new TeamResourceTest();
TEAM1 = teamResourceTest.createEntity(teamResourceTest.createRequest(test), ADMIN_AUTH_HEADERS);
TEAM_OWNER1 = new EntityReference().withId(TEAM1.getId()).withType(Entity.TEAM);
TEAM_OWNER1 = new TeamEntityInterface(TEAM1).getEntityReference();
ROLE1 = roleResourceTest.createEntity(roleResourceTest.createRequest(test), ADMIN_AUTH_HEADERS);
ROLE1_REFERENCE = new EntityReference().withId(ROLE1.getId()).withType("role");
ROLE1_REFERENCE = new RoleEntityInterface(ROLE1).getEntityReference();
// Create snowflake database service
DatabaseServiceResourceTest databaseServiceResourceTest = new DatabaseServiceResourceTest();
@ -280,11 +290,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
.withDatabaseConnection(TestUtils.DATABASE_CONNECTION);
DatabaseService databaseService =
new DatabaseServiceResourceTest().createEntity(createDatabaseService, ADMIN_AUTH_HEADERS);
SNOWFLAKE_REFERENCE =
new EntityReference()
.withName(databaseService.getName())
.withId(databaseService.getId())
.withType(Entity.DATABASE_SERVICE);
SNOWFLAKE_REFERENCE = new DatabaseServiceEntityInterface(databaseService).getEntityReference();
createDatabaseService.withName("redshiftDB").withServiceType(DatabaseServiceType.Redshift);
databaseService = databaseServiceResourceTest.createEntity(createDatabaseService, ADMIN_AUTH_HEADERS);
@ -339,14 +345,12 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
CreateStorageService createService =
new CreateStorageService().withName("s3").withServiceType(StorageServiceType.S3);
StorageService service = storageServiceResourceTest.createEntity(createService, ADMIN_AUTH_HEADERS);
AWS_STORAGE_SERVICE_REFERENCE =
new EntityReference().withName(service.getName()).withId(service.getId()).withType(Entity.STORAGE_SERVICE);
AWS_STORAGE_SERVICE_REFERENCE = new StorageServiceEntityInterface(service).getEntityReference();
// Create GCP storage service, GCS
createService.withName("gs").withServiceType(StorageServiceType.GCS);
service = storageServiceResourceTest.createEntity(createService, ADMIN_AUTH_HEADERS);
GCP_STORAGE_SERVICE_REFERENCE =
new EntityReference().withName(service.getName()).withId(service.getId()).withType(Entity.STORAGE_SERVICE);
GCP_STORAGE_SERVICE_REFERENCE = new StorageServiceEntityInterface(service).getEntityReference();
USER_ADDRESS_TAG_LABEL = getTagLabel("User.Address");
USER_BANK_ACCOUNT_TAG_LABEL = getTagLabel("User.BankAccount");
@ -1149,7 +1153,6 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
entity = patchEntityAndCheck(entity, origJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
entityInterface = getEntityInterface(entity);
entityInterface.setOwner(TEAM_OWNER1); // Get rid of href and name in the response for owner
//
// Replace description, add tags tier, owner
@ -1183,8 +1186,6 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
entity = patchEntityAndCheck(entity, origJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
entityInterface = getEntityInterface(entity);
entityInterface.setOwner(USER_OWNER1); // Get rid of href and name in the response for owner
//
// Remove description, tier, owner
//
@ -1963,6 +1964,14 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
}
}
protected void assertTermReferences(List<TermReference> expectedList, List<TermReference> actualList) {
for (TermReference expected : expectedList) {
TermReference actual =
actualList.stream().filter(a -> EntityUtil.termReferenceMatch.test(a, expected)).findAny().orElse(null);
assertNotNull(actual, "Expected termReference " + expected + " not found");
}
}
public final String getEntityName(TestInfo test) {
return String.format("%s_%s", entityType, test.getDisplayName());
}

5
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookCallbackResource.java
View File

@ -1,12 +1,11 @@
package org.openmetadata.catalog.resources.events;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import javax.ws.rs.Consumes;
@ -128,7 +127,7 @@ public class WebhookCallbackResource {
}
public List<ChangeEvent> getEntityCallbackEvents(EventType eventType, String entity) {
return Optional.ofNullable(entityCallbackMap.get(eventType + ":" + entity)).orElse(Collections.emptyList());
return listOrEmpty(entityCallbackMap.get(eventType + ":" + entity));
}
public EventDetails getEventDetails(String endpoint) {

11
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryResourceTest.java
View File

@ -75,9 +75,16 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
change.getFieldsAdded().add(new FieldChange().withName("reviewers").withNewValue(List.of(USER_OWNER1)));
glossary = patchEntityAndCheck(glossary, origJson, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change);
// Remove a reviewer in PATCH request
// Add another reviewer USER2 in PATCH request
origJson = JsonUtils.pojoToJson(glossary);
glossary.withReviewers(null);
glossary.withReviewers(List.of(USER_OWNER1, USER_OWNER2));
change = getChangeDescription(glossary.getVersion());
change.getFieldsAdded().add(new FieldChange().withName("reviewers").withNewValue(List.of(USER_OWNER2)));
glossary = patchEntityAndCheck(glossary, origJson, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change);
// Remove a reviewer USER1 in PATCH request
origJson = JsonUtils.pojoToJson(glossary);
glossary.withReviewers(List.of(USER_OWNER2));
change = getChangeDescription(glossary.getVersion());
change.getFieldsDeleted().add(new FieldChange().withName("reviewers").withOldValue(List.of(USER_OWNER1)));
patchEntityAndCheck(glossary, origJson, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change);

45
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResourceTest.java
View File

@ -29,6 +29,7 @@ import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.validateEntityReference;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collections;
@ -45,8 +46,10 @@ import org.junit.jupiter.api.TestMethodOrder;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateGlossary;
import org.openmetadata.catalog.api.data.CreateGlossaryTerm;
import org.openmetadata.catalog.api.data.TermReference;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.entity.data.GlossaryTerm;
import org.openmetadata.catalog.entity.data.GlossaryTerm.Status;
import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface;
import org.openmetadata.catalog.jdbi3.GlossaryTermRepository.GlossaryTermEntityInterface;
import org.openmetadata.catalog.resources.EntityResourceTest;
@ -187,20 +190,44 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
CreateGlossaryTerm create = createRequest(getEntityName(test), "", "", null).withReviewers(null).withSynonyms(null);
GlossaryTerm term = createEntity(create, ADMIN_AUTH_HEADERS);
// Add reviewer USER1 in PATCH request
// Add reviewer USER1, synonym1, reference1 in PATCH request
String origJson = JsonUtils.pojoToJson(term);
term.withReviewers(List.of(USER_OWNER1)).withSynonyms(List.of("synonym1"));
TermReference reference1 = new TermReference().withName("reference1").withEndpoint(URI.create("http://reference1"));
term.withReviewers(List.of(USER_OWNER1)).withSynonyms(List.of("synonym1")).withReferences(List.of(reference1));
ChangeDescription change = getChangeDescription(term.getVersion());
change.getFieldsAdded().add(new FieldChange().withName("reviewers").withNewValue(List.of(USER_OWNER1)));
change.getFieldsAdded().add(new FieldChange().withName("synonyms").withNewValue(List.of("synonym1")));
change.getFieldsAdded().add(new FieldChange().withName("references").withNewValue(List.of(reference1)));
term = patchEntityAndCheck(term, origJson, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change);
// Remove a reviewer in PATCH request
// Add reviewer USER2, synonym2, reference2 in PATCH request
origJson = JsonUtils.pojoToJson(term);
term.withReviewers(null).withSynonyms(null);
TermReference reference2 = new TermReference().withName("reference2").withEndpoint(URI.create("http://reference2"));
term.withReviewers(List.of(USER_OWNER1, USER_OWNER2))
.withSynonyms(List.of("synonym1", "synonym2"))
.withReferences(List.of(reference1, reference2));
change = getChangeDescription(term.getVersion());
change.getFieldsAdded().add(new FieldChange().withName("reviewers").withNewValue(List.of(USER_OWNER2)));
change.getFieldsAdded().add(new FieldChange().withName("synonyms").withNewValue(List.of("synonym2")));
change.getFieldsAdded().add(new FieldChange().withName("references").withNewValue(List.of(reference2)));
term = patchEntityAndCheck(term, origJson, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change);
// Remove a reviewer USER1, synonym1, reference1 in PATCH request
origJson = JsonUtils.pojoToJson(term);
term.withReviewers(List.of(USER_OWNER2)).withSynonyms(List.of("synonym2")).withReferences(List.of(reference2));
change = getChangeDescription(term.getVersion());
change.getFieldsDeleted().add(new FieldChange().withName("reviewers").withOldValue(List.of(USER_OWNER1)));
change.getFieldsDeleted().add(new FieldChange().withName("synonyms").withOldValue(List.of("synonym1")));
change.getFieldsDeleted().add(new FieldChange().withName("references").withOldValue(List.of(reference1)));
term = patchEntityAndCheck(term, origJson, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change);
// Change GlossaryTerm status from DRAFT to Approved
origJson = JsonUtils.pojoToJson(term);
term.withStatus(Status.APPROVED);
change = getChangeDescription(term.getVersion());
change
.getFieldsUpdated()
.add(new FieldChange().withName("status").withOldValue(Status.DRAFT).withNewValue(Status.APPROVED));
patchEntityAndCheck(term, origJson, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change);
}
@ -317,6 +344,16 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
List<String> expectedRefs = (List<String>) expected;
List<String> actualRefs = JsonUtils.readObjects(actual.toString(), String.class);
assertStrings(expectedRefs, actualRefs);
} else if (fieldName.equals("references")) {
@SuppressWarnings("unchecked")
List<TermReference> expectedRefs = (List<TermReference>) expected;
List<TermReference> actualRefs = JsonUtils.readObjects(actual.toString(), TermReference.class);
assertTermReferences(expectedRefs, actualRefs);
} else if (fieldName.equals("status")) {
@SuppressWarnings("unchecked")
Status expectedStatus = (Status) expected;
Status actualStatus = Status.fromValue(actual.toString());
assertEquals(expectedStatus, actualStatus);
} else {
assertCommonFieldChange(fieldName, expected, actual);
}

6
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/tags/TagResourceTest.java
View File

@ -24,6 +24,7 @@ import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.catalog.util.TestUtils.TEST_AUTH_HEADERS;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.assertResponseContains;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
@ -31,7 +32,6 @@ import java.net.URI;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response.Status;
import lombok.extern.slf4j.Slf4j;
@ -478,7 +478,7 @@ public class TagResourceTest extends CatalogApplicationTest {
private void validate(TagCategory category) {
assertNotNull(category.getName());
assertEquals(URI.create(BASE_URL + "/" + category.getName()), category.getHref());
for (Tag tag : Optional.ofNullable(category.getChildren()).orElse(Collections.emptyList())) {
for (Tag tag : listOrEmpty(category.getChildren())) {
validateHRef(category.getHref().toString(), tag);
}
}
@ -506,7 +506,7 @@ public class TagResourceTest extends CatalogApplicationTest {
assertNotNull(actual.getName(), actual.getFullyQualifiedName());
String href = parentURI + "/" + actual.getName();
assertEquals(URI.create(href), actual.getHref());
for (Tag child : Optional.ofNullable(actual.getChildren()).orElse(Collections.emptyList())) {
for (Tag child : listOrEmpty(actual.getChildren())) {
validateHRef(actual.getHref().toString(), child);
}
}

22
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java
View File

@ -26,6 +26,7 @@ import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.assertListNotNull;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.validateEntityReferences;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
@ -35,7 +36,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
import java.util.UUID;
import java.util.stream.Collectors;
@ -279,7 +279,7 @@ public class TeamResourceTest extends EntityResourceTest<Team, CreateTeam> {
// Remove a user from the team using patch request
String json = JsonUtils.pojoToJson(team);
int removeUserIndex = new Random().nextInt(totalUsers);
EntityReference deletedUser = team.getUsers().get(removeUserIndex).withHref(null);
EntityReference deletedUser = team.getUsers().get(removeUserIndex);
team.getUsers().remove(removeUserIndex);
ChangeDescription change = getChangeDescription(team.getVersion());
change.getFieldsDeleted().add(new FieldChange().withName("users").withOldValue(Arrays.asList(deletedUser)));
@ -288,7 +288,7 @@ public class TeamResourceTest extends EntityResourceTest<Team, CreateTeam> {
// Remove a default role from the team using patch request
json = JsonUtils.pojoToJson(team);
int removeDefaultRoleIndex = new Random().nextInt(roles.size());
EntityReference deletedRole = team.getDefaultRoles().get(removeDefaultRoleIndex).withHref(null);
EntityReference deletedRole = team.getDefaultRoles().get(removeDefaultRoleIndex);
team.getDefaultRoles().remove(removeDefaultRoleIndex);
change = getChangeDescription(team.getVersion());
change.getFieldsDeleted().add(new FieldChange().withName("defaultRoles").withOldValue(Arrays.asList(deletedRole)));
@ -363,14 +363,14 @@ public class TeamResourceTest extends EntityResourceTest<Team, CreateTeam> {
TestUtils.validateEntityReferences(team.getOwns());
List<EntityReference> expectedUsers = new ArrayList<>();
for (UUID userId : Optional.ofNullable(createRequest.getUsers()).orElse(Collections.emptyList())) {
for (UUID userId : listOrEmpty(createRequest.getUsers())) {
expectedUsers.add(new EntityReference().withId(userId).withType(Entity.USER));
}
expectedUsers = expectedUsers.isEmpty() ? null : expectedUsers;
TestUtils.assertEntityReferenceList(expectedUsers, team.getUsers());
List<EntityReference> expectedDefaultRoles = new ArrayList<>();
for (UUID roleId : Optional.ofNullable(createRequest.getDefaultRoles()).orElse(Collections.emptyList())) {
for (UUID roleId : listOrEmpty(createRequest.getDefaultRoles())) {
expectedDefaultRoles.add(new EntityReference().withId(roleId).withType(Entity.ROLE));
}
expectedDefaultRoles = expectedDefaultRoles.isEmpty() ? null : expectedDefaultRoles;
@ -389,7 +389,7 @@ public class TeamResourceTest extends EntityResourceTest<Team, CreateTeam> {
super.validateDeletedEntity(create, teamBeforeDeletion, teamAfterDeletion, authHeaders);
List<EntityReference> expectedOwnedEntities = new ArrayList<>();
for (EntityReference ref : Optional.ofNullable(teamBeforeDeletion.getOwns()).orElse(Collections.emptyList())) {
for (EntityReference ref : listOrEmpty(teamBeforeDeletion.getOwns())) {
expectedOwnedEntities.add(new EntityReference().withId(ref.getId()).withType(Entity.TABLE));
}
TestUtils.assertEntityReferenceList(expectedOwnedEntities, teamAfterDeletion.getOwns());
@ -404,14 +404,12 @@ public class TeamResourceTest extends EntityResourceTest<Team, CreateTeam> {
assertEquals(expected.getProfile(), updated.getProfile());
TestUtils.validateEntityReferences(updated.getOwns());
List<EntityReference> expectedUsers = Optional.ofNullable(expected.getUsers()).orElse(Collections.emptyList());
List<EntityReference> actualUsers = Optional.ofNullable(updated.getUsers()).orElse(Collections.emptyList());
List<EntityReference> expectedUsers = listOrEmpty(expected.getUsers());
List<EntityReference> actualUsers = listOrEmpty(updated.getUsers());
TestUtils.assertEntityReferenceList(expectedUsers, actualUsers);
List<EntityReference> expectedDefaultRoles =
Optional.ofNullable(expected.getDefaultRoles()).orElse(Collections.emptyList());
List<EntityReference> actualDefaultRoles =
Optional.ofNullable(updated.getDefaultRoles()).orElse(Collections.emptyList());
List<EntityReference> expectedDefaultRoles = listOrEmpty(expected.getDefaultRoles());
List<EntityReference> actualDefaultRoles = listOrEmpty(updated.getDefaultRoles());
TestUtils.assertEntityReferenceList(expectedDefaultRoles, actualDefaultRoles);
}

12
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/UserResourceTest.java
View File

@ -36,6 +36,7 @@ import static org.openmetadata.catalog.util.TestUtils.assertListNotNull;
import static org.openmetadata.catalog.util.TestUtils.assertListNull;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.assertResponseContains;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
@ -46,7 +47,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@ -684,7 +684,7 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
super.validateDeletedEntity(create, userBeforeDeletion, userAfterDeletion, authHeaders);
List<EntityReference> expectedOwnedEntities = new ArrayList<>();
for (EntityReference ref : Optional.ofNullable(userBeforeDeletion.getOwns()).orElse(Collections.emptyList())) {
for (EntityReference ref : listOrEmpty(userBeforeDeletion.getOwns())) {
expectedOwnedEntities.add(new EntityReference().withId(ref.getId()).withType(Entity.TABLE));
}
@ -703,7 +703,7 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
assertEquals(createRequest.getIsAdmin(), user.getIsAdmin());
List<EntityReference> expectedRoles = new ArrayList<>();
for (UUID roleId : Optional.ofNullable(createRequest.getRoles()).orElse(Collections.emptyList())) {
for (UUID roleId : listOrEmpty(createRequest.getRoles())) {
expectedRoles.add(new EntityReference().withId(roleId).withType(Entity.ROLE));
}
if (expectedRoles.isEmpty()) {
@ -714,7 +714,7 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
}
List<EntityReference> expectedTeams = new ArrayList<>();
for (UUID teamId : Optional.ofNullable(createRequest.getTeams()).orElse(Collections.emptyList())) {
for (UUID teamId : listOrEmpty(createRequest.getTeams())) {
expectedTeams.add(new EntityReference().withId(teamId).withType(Entity.TEAM));
}
TestUtils.assertEntityReferenceList(expectedTeams, user.getTeams());
@ -743,8 +743,8 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
}
private void compareEntityReferenceLists(List<EntityReference> expected, List<EntityReference> updated) {
List<EntityReference> expectedList = Optional.ofNullable(expected).orElse(Collections.emptyList());
List<EntityReference> updatedList = new ArrayList<>(Optional.ofNullable(updated).orElse(Collections.emptyList()));
List<EntityReference> expectedList = listOrEmpty(expected);
List<EntityReference> updatedList = new ArrayList<>(listOrEmpty(updated));
updatedList.forEach(TestUtils::validateEntityReference);
expectedList.sort(EntityUtil.compareEntityReference);

9
catalog-rest-service/src/test/java/org/openmetadata/catalog/util/TestUtils.java
View File

@ -19,14 +19,13 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.catalog.security.SecurityUtil.authHeaders;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.json.JsonObject;
@ -227,21 +226,21 @@ public final class TestUtils {
}
public static void validateEntityReferences(List<EntityReference> list) {
Optional.ofNullable(list).orElse(Collections.emptyList()).forEach(TestUtils::validateEntityReference);
listOrEmpty(list).forEach(TestUtils::validateEntityReference);
}
public static void validateTags(List<TagLabel> expectedList, List<TagLabel> actualList) throws HttpResponseException {
if (expectedList == null) {
return;
}
actualList = Optional.ofNullable(actualList).orElse(Collections.emptyList());
actualList = listOrEmpty(actualList);
// When tags from the expected list is added to an entity, the derived tags for those tags are automatically added
// So add to the expectedList, the derived tags before validating the tags
List<TagLabel> updatedExpectedList = new ArrayList<>(expectedList);
for (TagLabel expected : expectedList) {
Tag tag = TagResourceTest.getTag(expected.getTagFQN(), ADMIN_AUTH_HEADERS);
List<TagLabel> derived = new ArrayList<>();
for (String fqn : Optional.ofNullable(tag.getAssociatedTags()).orElse(Collections.emptyList())) {
for (String fqn : listOrEmpty(tag.getAssociatedTags())) {
Tag associatedTag = TagResourceTest.getTag(fqn, ADMIN_AUTH_HEADERS);
derived.add(
new TagLabel()

6
common/src/main/java/org/openmetadata/common/utils/CommonUtil.java
View File

@ -25,9 +25,11 @@ import java.util.ArrayList;
import java.util.Base64;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.Enumeration;
import java.util.List;
import java.util.Optional;
import java.util.TimeZone;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@ -152,4 +154,8 @@ public final class CommonUtil {
throw new RuntimeException("Failed to calculate " + HMAC_SHA256_ALGORITHM, e);
}
}
public static <T> List<T> listOrEmpty(List<T> list) {
return Optional.ofNullable(list).orElse(Collections.emptyList());
}
}