diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index a2d0fe600fb..4c0a90b7aea 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -44,6 +44,7 @@ import lombok.Getter; import lombok.NonNull; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.jdbi.v3.core.Jdbi; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.EntityTimeSeriesInterface; @@ -397,6 +398,13 @@ public final class Entity { return repository.getReference(id, include); } + public static List getEntityReferencesByIds( + @NonNull String entityType, @NonNull List ids, Include include) { + EntityRepository repository = getEntityRepository(entityType); + include = repository.supportsSoftDelete ? Include.ALL : include; + return repository.getReferences(ids, include); + } + public static EntityReference getEntityReferenceByName( @NonNull String entityType, String fqn, Include include) { if (fqn == null) { @@ -437,6 +445,23 @@ public final class Entity { : getEntityByName(ref.getType(), ref.getFullyQualifiedName(), fields, include); } + public static List getEntities( + List refs, String fields, Include include) { + if (CollectionUtils.isEmpty(refs)) { + return new ArrayList<>(); + } + EntityRepository entityRepository = Entity.getEntityRepository(refs.get(0).getType()); + @SuppressWarnings("unchecked") + List entities = + (List) + entityRepository.get( + null, + refs.stream().map(EntityReference::getId).toList(), + entityRepository.getFields(fields), + include); + return entities; + } + public static T getEntityOrNull( EntityReference entityReference, String field, Include include) { if (entityReference == null) return null; @@ -478,6 +503,16 @@ public final class Entity { return getEntityByName(entityType, fqn, fields, include, true); } + public static List getEntityByNames( + String entityType, List tagFQNs, String fields, Include include) { + EntityRepository entityRepository = Entity.getEntityRepository(entityType); + @SuppressWarnings("unchecked") + List entities = + (List) + entityRepository.getByNames(null, tagFQNs, entityRepository.getFields(fields), include); + return entities; + } + /** Retrieve the corresponding entity repository for a given entity name. */ public static EntityRepository getEntityRepository( @NonNull String entityType) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index c40fcd4f34e..58d0bf5a7dc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -876,6 +876,13 @@ public interface CollectionDAO { private String json; } + @Getter + @Builder + class EntityRelationshipCount { + private UUID id; + private Integer count; + } + @Getter @Builder class EntityRelationshipObject { @@ -1009,6 +1016,19 @@ public interface CollectionDAO { @Bind("fromEntityType") String fromEntityType, @Bind("toEntityType") String toEntityType); + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation " + + "FROM entity_relationship " + + "WHERE fromId IN () " + + "AND relation = :relation " + + "AND toEntity = :toEntityType " + + "AND deleted = FALSE") + @UseRowMapper(RelationshipObjectMapper.class) + List findToBatch( + @BindList("fromIds") List fromIds, + @Bind("relation") int relation, + @Bind("toEntityType") String toEntityType); + @SqlQuery( "SELECT toId, toEntity, json FROM entity_relationship " + "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity") @@ -1019,6 +1039,17 @@ public interface CollectionDAO { @Bind("relation") int relation, @Bind("toEntity") String toEntity); + @SqlQuery( + "SELECT fromId, COUNT(toId) FROM entity_relationship " + + "WHERE fromId IN () AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity " + + "GROUP BY fromId") + @RegisterRowMapper(ToRelationshipCountMapper.class) + List countFindTo( + @BindList("fromIds") List fromIds, + @Bind("fromEntity") String fromEntity, + @Bind("relation") int relation, + @Bind("toEntity") String toEntity); + @SqlQuery( "SELECT COUNT(toId) FROM entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity " + "AND relation IN ()") @@ -1300,6 +1331,16 @@ public interface CollectionDAO { } } + class ToRelationshipCountMapper implements RowMapper { + @Override + public EntityRelationshipCount map(ResultSet rs, StatementContext ctx) throws SQLException { + return EntityRelationshipCount.builder() + .id(UUID.fromString(rs.getString(1))) + .count(rs.getInt(2)) + .build(); + } + } + class RelationshipObjectMapper implements RowMapper { @Override public EntityRelationshipObject map(ResultSet rs, StatementContext ctx) throws SQLException { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java index 2fcbfb97c28..16a0771d59c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java @@ -19,11 +19,18 @@ import static org.openmetadata.service.jdbi3.ListFilter.escapeApostrophe; import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.List; import java.util.Map; import java.util.UUID; import lombok.SneakyThrows; +import org.apache.commons.collections4.CollectionUtils; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindList; import org.jdbi.v3.sqlobject.customizer.BindMap; import org.jdbi.v3.sqlobject.customizer.Define; import org.jdbi.v3.sqlobject.statement.BatchChunkSize; @@ -148,6 +155,13 @@ public interface EntityDAO { String findById( @Define("table") String table, @BindUUID("id") UUID id, @Define("cond") String cond); + @SqlQuery("SELECT id, json FROM WHERE id IN () ") + @RegisterRowMapper(EntityIdJsonPairMapper.class) + List findByIds( + @Define("table") String table, + @BindList("ids") List ids, + @Define("cond") String cond); + @SqlQuery("SELECT json FROM
WHERE = :name ") String findByName( @Define("table") String table, @@ -155,6 +169,14 @@ public interface EntityDAO { @BindFQN("name") String name, @Define("cond") String cond); + @SqlQuery("SELECT , json FROM
WHERE IN () ") + @RegisterRowMapper(EntityNameColumnHashJsonPairMapper.class) + List findByNames( + @Define("table") String table, + @Define("nameColumnHash") String nameColumn, + @BindList("names") List names, + @Define("cond") String cond); + @SqlQuery("SELECT count() FROM
") int listCount( @Define("table") String table, @@ -448,6 +470,19 @@ public interface EntityDAO { return findEntityById(id, Include.NON_DELETED); } + default List findEntitiesByIds(List ids, Include include) { + if (CollectionUtils.isEmpty(ids)) { + return List.of(); + } + return findByIds( + getTableName(), + ids.stream().map(UUID::toString).distinct().toList(), + getCondition(include)) + .stream() + .map(pair -> jsonToEntity(pair.json, pair.id)) + .toList(); + } + default T findEntityByName(String fqn) { return findEntityByName(fqn, Include.NON_DELETED); } @@ -464,6 +499,17 @@ public interface EntityDAO { findByName(getTableName(), nameHashColumn, fqn, getCondition(include)), fqn); } + @SneakyThrows + default List findEntityByNames(List entityFQNs, Include include) { + if (CollectionUtils.isEmpty(entityFQNs)) { + return List.of(); + } + List names = entityFQNs.stream().distinct().map(FullyQualifiedName::buildHash).toList(); + return findByNames(getTableName(), getNameHashColumn(), names, getCondition(include)).stream() + .map(pair -> jsonToEntity(pair.json, pair.nameColumnHash)) + .toList(); + } + default T jsonToEntity(String json, Object identity) { Class clz = getEntityClass(); T entity = json != null ? JsonUtils.readValue(json, clz) : null; @@ -549,4 +595,22 @@ public interface EntityDAO { throw EntityNotFoundException.byMessage(entityNotFound(entityType, id)); } } + + record EntityNameColumnHashJsonPair(String nameColumnHash, String json) {} + + class EntityNameColumnHashJsonPairMapper implements RowMapper { + @Override + public EntityNameColumnHashJsonPair map(ResultSet r, StatementContext ctx) throws SQLException { + return new EntityNameColumnHashJsonPair(r.getString(1), r.getString(2)); + } + } + + record EntityIdJsonPair(UUID id, String json) {} + + class EntityIdJsonPairMapper implements RowMapper { + @Override + public EntityIdJsonPair map(ResultSet r, StatementContext ctx) throws SQLException { + return new EntityIdJsonPair(UUID.fromString(r.getString(1)), r.getString(2)); + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index c0bb092adbd..92aba4d5302 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -120,6 +120,7 @@ import javax.ws.rs.core.UriInfo; import lombok.Getter; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.jdbi.v3.sqlobject.transaction.Transaction; @@ -132,6 +133,7 @@ import org.openmetadata.schema.api.VoteRequest.VoteType; import org.openmetadata.schema.api.feed.ResolveTask; import org.openmetadata.schema.api.teams.CreateTeam; import org.openmetadata.schema.configuration.AssetCertificationSettings; +import org.openmetadata.schema.entity.classification.Tag; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.feed.Suggestion; import org.openmetadata.schema.entity.teams.Team; @@ -257,6 +259,7 @@ public abstract class EntityRepository { @Getter protected final boolean supportsStyle; @Getter protected final boolean supportsLifeCycle; @Getter protected final boolean supportsCertification; + @Getter protected final boolean supportsChildren; protected final boolean supportsFollower; protected final boolean supportsExtension; protected final boolean supportsVotes; @@ -373,6 +376,7 @@ public abstract class EntityRepository { this.patchFields.addField(allowedFields, FIELD_CERTIFICATION); this.putFields.addField(allowedFields, FIELD_CERTIFICATION); } + this.supportsChildren = allowedFields.contains(FIELD_CHILDREN); Map, Fields>>> fieldSupportMap = new HashMap<>(); @@ -381,6 +385,7 @@ public abstract class EntityRepository { fieldSupportMap.put(FIELD_DOMAIN, Pair.of(supportsDomain, this::fetchAndSetDomain)); fieldSupportMap.put(FIELD_REVIEWERS, Pair.of(supportsReviewers, this::fetchAndSetReviewers)); fieldSupportMap.put(FIELD_EXTENSION, Pair.of(supportsExtension, this::fetchAndSetExtension)); + fieldSupportMap.put(FIELD_CHILDREN, Pair.of(supportsChildren, this::fetchAndSetChildren)); for (Entry, Fields>>> entry : fieldSupportMap.entrySet()) { @@ -390,8 +395,6 @@ public abstract class EntityRepository { if (supportsField) { this.fieldFetchers.put(fieldName, fetcher); - this.patchFields.addField(allowedFields, fieldName); - this.putFields.addField(allowedFields, fieldName); } } @@ -474,7 +477,11 @@ public abstract class EntityRepository { } } - @SuppressWarnings("unused") + /** + * The default behavior is to execute one by one. For batch execution, override this method in the subclass. + * + * @see GlossaryTermRepository#setInheritedFields(List, Fields) for an example implementation + */ protected void setInheritedFields(List entities, Fields fields) { for (T entity : entities) { setInheritedFields(entity, fields); @@ -633,6 +640,13 @@ public abstract class EntityRepository { return withHref(uriInfo, entityClone); } + public final List get(UriInfo uriInfo, List ids, Fields fields, Include include) { + List entities = find(ids, include); + setFieldsInBulk(fields, entities); + entities.forEach(entity -> withHref(uriInfo, entity)); + return entities; + } + /** * getReference is used for getting the entity references from the entity in the cache. */ @@ -641,6 +655,11 @@ public abstract class EntityRepository { return find(id, include).getEntityReference(); } + public final List getReferences(List id, Include include) + throws EntityNotFoundException { + return find(id, include).stream().map(EntityInterface::getEntityReference).toList(); + } + /** * Find method is used for getting an entity only with core fields stored as JSON without any relational fields set */ @@ -665,6 +684,10 @@ public abstract class EntityRepository { } } + public final List find(List ids, Include include) { + return dao.findEntitiesByIds(ids, include); + } + public T getByName(UriInfo uriInfo, String fqn, Fields fields) { return getByName(uriInfo, fqn, fields, NON_DELETED, false); } @@ -695,6 +718,14 @@ public abstract class EntityRepository { return findByName(fqn, include).getEntityReference(); } + public final List getByNames( + UriInfo uriInfo, List entityFQNs, Fields fields, Include include) { + List entities = findByNames(entityFQNs, include); + setFieldsInBulk(fields, entities); + entities.forEach(entity -> withHref(uriInfo, entity)); + return entities; + } + public final T findByNameOrNull(String fqn, Include include) { try { return findByName(fqn, include); @@ -728,16 +759,19 @@ public abstract class EntityRepository { } } + public List findByNames(List entityFQNs, Include include) { + return dao.findEntityByNames(entityFQNs, include); + } + public final List listAll(Fields fields, ListFilter filter) { // forward scrolling, if after == null then first page is being asked List jsons = dao.listAfter(filter, Integer.MAX_VALUE, "", ""); List entities = new ArrayList<>(); for (String json : jsons) { - T entity = setFieldsInternal(JsonUtils.readValue(json, entityClass), fields); - setInheritedFields(entity, fields); - clearFieldsInternal(entity, fields); + T entity = JsonUtils.readValue(json, entityClass); entities.add(entity); } + setFieldsInBulk(fields, entities); return entities; } @@ -745,17 +779,32 @@ public abstract class EntityRepository { ListFilter filter = new ListFilter(NON_DELETED); List jsons = listAllByParentFqn(parentFqn, filter); List entities = new ArrayList<>(); - setFieldsInBulk(jsons, fields, entities); - return entities; - } - - public void setFieldsInBulk(List jsons, Fields fields, List entities) { for (String json : jsons) { T entity = JsonUtils.readValue(json, entityClass); entities.add(entity); } + // TODO: Ensure consistent behavior with setFieldsInBulk when all repositories implement it fetchAndSetFields(entities, fields); + setInheritedFields(entities, fields); for (T entity : entities) { + clearFieldsInternal(entity, fields); + } + return entities; + } + + /** + * Executes {@link #setFields} on a list of entities. By default, this method processes + * each entity individually. To enable batch processing, override this method in a subclass. + *

+ * For efficient bulk processing, ensure all fields used in {@link #setFields} + * have corresponding batch processing methods, such as {@code fetchAndSetXXX}. For instance, + * if handling a domain field, implement {@link #fetchAndSetDomain}. + *

+ * Example implementation can be found in {@link GlossaryTermRepository#setFieldsInBulk}. + */ + public void setFieldsInBulk(Fields fields, List entities) { + for (T entity : entities) { + setFieldsInternal(entity, fields); setInheritedFields(entity, fields); clearFieldsInternal(entity, fields); } @@ -788,11 +837,11 @@ public abstract class EntityRepository { List jsons = dao.listAfter(filter, limitParam + 1, afterName, afterId); for (String json : jsons) { - T entity = setFieldsInternal(JsonUtils.readValue(json, entityClass), fields); - setInheritedFields(entity, fields); - clearFieldsInternal(entity, fields); - entities.add(withHref(uriInfo, entity)); + T entity = JsonUtils.readValue(json, entityClass); + entities.add(entity); } + setFieldsInBulk(fields, entities); + entities.forEach(entity -> withHref(uriInfo, entity)); String beforeCursor; String afterCursor = null; @@ -834,11 +883,12 @@ public abstract class EntityRepository { List entities = new ArrayList<>(); for (String json : jsons) { - T entity = setFieldsInternal(JsonUtils.readValue(json, entityClass), fields); - setInheritedFields(entity, fields); - clearFieldsInternal(entity, fields); - entities.add(withHref(uriInfo, entity)); + T entity = JsonUtils.readValue(json, entityClass); + entities.add(entity); } + setFieldsInBulk(fields, entities); + entities.forEach(entity -> withHref(uriInfo, entity)); + int total = dao.listCount(filter); String beforeCursor = null; @@ -3943,13 +3993,8 @@ public abstract class EntityRepository { } } - private void fetchAndSetFields(List entities, Fields fields) { - for (String field : fields) { - BiConsumer, Fields> fetcher = fieldFetchers.get(field); - if (fetcher != null) { - fetcher.accept(entities, fields); - } - } + protected void fetchAndSetFields(List entities, Fields fields) { + fieldFetchers.values().forEach(fetcher -> fetcher.accept(entities, fields)); } private void fetchAndSetOwners(List entities, Fields fields) { @@ -4008,6 +4053,18 @@ public abstract class EntityRepository { } } + protected void fetchAndSetChildren(List entities, Fields fields) { + if (!fields.contains(FIELD_CHILDREN) || entities == null || entities.isEmpty()) { + return; + } + + Map> childrenMap = batchFetchChildren(entities); + + for (T entity : entities) { + entity.setChildren(childrenMap.get(entity.getId())); + } + } + private void fetchAndSetReviewers(List entities, Fields fields) { if (!fields.contains(FIELD_REVIEWERS) || !supportsReviewers || entities.isEmpty()) { return; @@ -4051,6 +4108,15 @@ public abstract class EntityRepository { List tags = daoCollection.tagUsageDAO().getTagsInternalBatch(entityFQNs); + List tagFQNs = + tags.stream() + .distinct() + .map(CollectionDAO.TagUsageDAO.TagLabelWithFQNHash::getTagFQN) + .toList(); + Map tagFQNLabelMap = + EntityUtil.toTagLabels(TagLabelUtil.getTags(tagFQNs).toArray(new Tag[0])).stream() + .collect(Collectors.toMap(TagLabel::getTagFQN, Function.identity())); + // Map from targetFQNHash to targetFQN Map fqnHashToFqnMap = entityFQNs.stream().collect(Collectors.toMap(FullyQualifiedName::buildHash, fqn -> fqn)); @@ -4062,7 +4128,8 @@ public abstract class EntityRepository { String targetFQN = fqnHashToFqnMap.get(targetFQNHash); if (targetFQN != null) { - tagsMap.computeIfAbsent(targetFQN, k -> new ArrayList<>()).add(tagWithHash.toTagLabel()); + TagLabel tagLabel = tagFQNLabelMap.get(tagWithHash.getTagFQN()); + tagsMap.computeIfAbsent(targetFQN, k -> new ArrayList<>()).add(tagLabel); } } @@ -4152,6 +4219,36 @@ public abstract class EntityRepository { return result; } + private Map> batchFetchChildren(List entities) { + List records = + daoCollection + .relationshipDAO() + .findToBatch( + entityListToStrings(entities), Relationship.CONTAINS.ordinal(), entityType); + + Map> childrenMap = new HashMap<>(); + + if (CollectionUtils.isEmpty(records)) { + return childrenMap; + } + + Map idReferenceMap = + Entity.getEntityReferencesByIds( + records.get(0).getToEntity(), + records.stream().map(e -> UUID.fromString(e.getToId())).distinct().toList(), + ALL) + .stream() + .collect(Collectors.toMap(e -> e.getId().toString(), Function.identity())); + + for (CollectionDAO.EntityRelationshipObject rec : records) { + UUID entityId = UUID.fromString(rec.getFromId()); + EntityReference childrenRef = idReferenceMap.get(rec.getToId()); + childrenMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(childrenRef); + } + + return childrenMap; + } + private List entityListToStrings(List entities) { return entities.stream().map(EntityInterface::getId).map(UUID::toString).toList(); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 31d0514cd3d..3fedf275921 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -41,6 +41,7 @@ import static org.openmetadata.service.util.EntityUtil.termReferenceMatch; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.google.gson.Gson; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -57,6 +58,7 @@ import java.util.stream.Collectors; import javax.json.JsonPatch; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.common.utils.CommonUtil; @@ -121,8 +123,10 @@ public class GlossaryTermRepository extends EntityRepository { UPDATE_FIELDS); supportsSearch = true; renameAllowed = true; - fieldFetchers.put("relatedTerms", this::fetchAndSetRelatedTerms); fieldFetchers.put("parent", this::fetchAndSetParentOrGlossary); + fieldFetchers.put("relatedTerms", this::fetchAndSetRelatedTerms); + fieldFetchers.put("usageCount", this::fetchAndSetUsageCount); + fieldFetchers.put("childrenCount", this::fetchAndSetChildrenCount); } @Override @@ -136,6 +140,15 @@ public class GlossaryTermRepository extends EntityRepository { fields.contains("childrenCount") ? getChildrenCount(entity) : entity.getChildrenCount()); } + @Override + public void setFieldsInBulk(Fields fields, List entities) { + fetchAndSetFields(entities, fields); + setInheritedFields(entities, fields); + for (GlossaryTerm entity : entities) { + clearFieldsInternal(entity, fields); + } + } + @Override public void clearFields(GlossaryTerm entity, Fields fields) { entity.setRelatedTerms(fields.contains("relatedTerms") ? entity.getRelatedTerms() : null); @@ -151,6 +164,25 @@ public class GlossaryTermRepository extends EntityRepository { inheritReviewers(glossaryTerm, fields, parent); } + @Override + public void setInheritedFields(List glossaryTerms, Fields fields) { + List parents = + getParentEntities(glossaryTerms, "owners,domain,reviewers"); + Map parentMap = + parents.stream().collect(Collectors.toMap(EntityInterface::getId, e -> e)); + for (GlossaryTerm glossaryTerm : glossaryTerms) { + EntityInterface parent = null; + if (glossaryTerm.getParent() != null) { + parent = parentMap.get(glossaryTerm.getParent().getId()); + } else if (glossaryTerm.getGlossary() != null) { + parent = parentMap.get(glossaryTerm.getGlossary().getId()); + } + inheritOwners(glossaryTerm, fields, parent); + inheritDomain(glossaryTerm, fields, parent); + inheritReviewers(glossaryTerm, fields, parent); + } + } + private Integer getUsageCount(GlossaryTerm term) { return daoCollection .tagUsageDAO() @@ -218,6 +250,24 @@ public class GlossaryTermRepository extends EntityRepository { .withReviewers(reviewers); } + @Override + public void storeEntities(List entities) { + List entitiesToStore = new ArrayList<>(); + Gson gson = new Gson(); + for (GlossaryTerm entity : entities) { + EntityReference glossary = entity.getGlossary(); + EntityReference parentTerm = entity.getParent(); + List reviewers = entity.getReviewers(); + + String jsonCopy = gson.toJson(entity.withGlossary(null).withParent(null).withReviewers(null)); + entitiesToStore.add(gson.fromJson(jsonCopy, GlossaryTerm.class)); + + // restore the relationships + entity.withGlossary(glossary).withParent(parentTerm).withReviewers(reviewers); + } + storeMany(entitiesToStore); + } + @Override public void storeRelationships(GlossaryTerm entity) { addGlossaryRelationship(entity); @@ -625,6 +675,24 @@ public class GlossaryTermRepository extends EntityRepository { : Entity.getEntity(entity.getGlossary(), fields, Include.ALL); } + public List getParentEntities(List entities, String fields) { + List result = new ArrayList<>(); + if (CollectionUtils.isEmpty(entities)) { + return result; + } + List parents = + entities.stream().map(GlossaryTerm::getParent).filter(Objects::nonNull).distinct().toList(); + result.addAll(Entity.getEntities(parents, fields, Include.ALL)); + List glossaries = + entities.stream() + .map(GlossaryTerm::getGlossary) + .filter(Objects::nonNull) + .distinct() + .toList(); + result.addAll(Entity.getEntities(glossaries, fields, Include.ALL)); + return result; + } + private void addGlossaryRelationship(GlossaryTerm term) { Relationship relationship = term.getParent() != null ? Relationship.HAS : Relationship.CONTAINS; addRelationship( @@ -841,11 +909,12 @@ public class GlossaryTermRepository extends EntityRepository { } private void fetchAndSetParentOrGlossary(List terms, Fields fields) { - if (terms == null || terms.isEmpty() || (!fields.contains("parent"))) { + if (terms == null || terms.isEmpty()) { return; } - List entityIds = terms.stream().map(GlossaryTerm::getId).map(UUID::toString).toList(); + List entityIds = + terms.stream().map(GlossaryTerm::getId).map(UUID::toString).distinct().toList(); List parentRecords = daoCollection @@ -919,6 +988,39 @@ public class GlossaryTermRepository extends EntityRepository { } } + private void fetchAndSetUsageCount(List entities, Fields fields) { + if (!fields.contains("usageCount") || entities.isEmpty()) { + return; + } + // TODO: modify to use a single db query + for (GlossaryTerm entity : entities) { + entity.withUsageCount(getUsageCount(entity)); + } + } + + private void fetchAndSetChildrenCount(List entities, Fields fields) { + if (!fields.contains("childrenCount") || entities.isEmpty()) { + return; + } + List termIds = + entities.stream().map(GlossaryTerm::getId).map(UUID::toString).distinct().toList(); + + Map termIdCountMap = + daoCollection + .relationshipDAO() + .countFindTo(termIds, GLOSSARY_TERM, Relationship.CONTAINS.ordinal(), GLOSSARY_TERM) + .stream() + .collect( + Collectors.toMap( + CollectionDAO.EntityRelationshipCount::getId, + CollectionDAO.EntityRelationshipCount::getCount)); + + for (GlossaryTerm entity : entities) { + entity.setChildrenCount( + termIdCountMap.getOrDefault(entity.getId(), entity.getChildrenCount())); + } + } + /** Handles entity updated from PUT and POST operation. */ public class GlossaryTermUpdater extends EntityUpdater { public GlossaryTermUpdater(GlossaryTerm original, GlossaryTerm updated, Operation operation) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java index 11ffd396f8b..5e90990a4ab 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java @@ -416,6 +416,36 @@ public class GlossaryTermResource extends EntityResource creates) { + List terms = + creates.stream() + .map( + create -> + mapper.createToEntity(create, securityContext.getUserPrincipal().getName())) + .toList(); + List result = repository.createMany(uriInfo, terms); + return Response.ok(result).build(); + } + @PATCH @Path("/{id}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java index 1fa10db732e..fdd952882f8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java @@ -52,6 +52,10 @@ public class TagLabelUtil { return Entity.getEntityByName(Entity.TAG, tagFqn, "", NON_DELETED); } + public static List getTags(List tagFQNs) { + return Entity.getEntityByNames(Entity.TAG, tagFQNs, "", NON_DELETED); + } + public static Glossary getGlossary(String glossaryName) { return Entity.getEntityByName(Entity.GLOSSARY, glossaryName, "", NON_DELETED); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java index 99c1bac7c52..262384f05ba 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java @@ -20,6 +20,7 @@ import static java.util.Collections.emptyList; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.NOT_FOUND; +import static javax.ws.rs.core.Response.Status.OK; import static org.junit.jupiter.api.Assertions.*; import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; @@ -56,6 +57,8 @@ import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.function.Predicate; +import java.util.stream.IntStream; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; @@ -1057,6 +1060,21 @@ public class GlossaryTermResourceTest extends EntityResourceTest> result = + createTerms(glossary, IntStream.range(0, 500).mapToObj(i -> "term" + i).toList()); + + Map queryParams = new HashMap<>(); + queryParams.put("fields", "children,relatedTerms,reviewers,tags"); + queryParams.put("limit", "10000"); + queryParams.put("directChildrenOf", "词汇表三"); + ResultList list = + assertTimeout(Duration.ofSeconds(3), () -> listEntities(queryParams, ADMIN_AUTH_HEADERS)); + assertEquals(result.size(), list.getData().size()); + } + public GlossaryTerm createTerm(Glossary glossary, GlossaryTerm parent, String termName) throws IOException { return createTerm(glossary, parent, termName, glossary.getReviewers()); @@ -1086,6 +1104,25 @@ public class GlossaryTermResourceTest extends EntityResourceTest> createTerms(Glossary glossary, List termNames) + throws HttpResponseException { + String pathUrl = "/createMany/"; + String glossaryFqn = getFqn(glossary); + WebTarget target = getCollection().path(pathUrl); + List createGlossaryTerms = + termNames.stream() + .map( + name -> + createRequest(name, "d", "", null) + .withRelatedTerms(null) + .withSynonyms(List.of("performance1", "performance2")) + .withStyle(new Style().withColor("#FF5733").withIconURL("https://img")) + .withGlossary(glossaryFqn)) + .toList(); + return TestUtils.post( + target, createGlossaryTerms, List.class, OK.getStatusCode(), ADMIN_AUTH_HEADERS); + } + public void assertContains(List expectedTerms, List actualTerms) throws HttpResponseException { assertEquals(expectedTerms.size(), actualTerms.size()); @@ -1341,6 +1378,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest queryParams = new HashMap<>(); queryParams.put("directChildrenOf", term1.getFullyQualifiedName()); + queryParams.put("fields", "childrenCount,children"); List children = listEntities(queryParams, ADMIN_AUTH_HEADERS).getData(); assertEquals(term1.getChildren().size(), children.size()); @@ -1348,10 +1386,21 @@ public class GlossaryTermResourceTest extends EntityResourceTest(); + queryParams.put("directChildrenOf", glossary1.getFullyQualifiedName()); + queryParams.put("fields", "childrenCount"); + children = listEntities(queryParams, ADMIN_AUTH_HEADERS).getData(); + assertEquals(term1.getChildren().size(), children.get(0).getChildrenCount()); } public Glossary createGlossary(