mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-08-27 10:26:09 +00:00
* Glossary Page Performance Issue with Large Number of Terms(#19761) * Add glossary page performance unittest(#19761) * Fix dup setFieldsInternal(#19761) * Add fetch childrenCount (#19761) * add comment about setFieldsInBulk(#19761) * rename to countFindTo(#19761) * fix findByIds return both id and json (#19761) * turn up UT timeout (#19761) --------- Co-authored-by: sonika-shah <58761340+sonika-shah@users.noreply.github.com>
This commit is contained in:
Juntao Zhang
GitHub
parent
9e13ff6be8
commit
67a0795f7b
@ -44,6 +44,7 @@ import lombok.Getter;
|
||||
import lombok.NonNull;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.jdbi.v3.core.Jdbi;
|
||||
import org.openmetadata.schema.EntityInterface;
|
||||
import org.openmetadata.schema.EntityTimeSeriesInterface;
|
||||
@ -397,6 +398,13 @@ public final class Entity {
|
||||
return repository.getReference(id, include);
|
||||
}
|
||||
|
||||
public static List<EntityReference> getEntityReferencesByIds(
|
||||
@NonNull String entityType, @NonNull List<UUID> ids, Include include) {
|
||||
EntityRepository<? extends EntityInterface> repository = getEntityRepository(entityType);
|
||||
include = repository.supportsSoftDelete ? Include.ALL : include;
|
||||
return repository.getReferences(ids, include);
|
||||
}
|
||||
|
||||
public static EntityReference getEntityReferenceByName(
|
||||
@NonNull String entityType, String fqn, Include include) {
|
||||
if (fqn == null) {
|
||||
@ -437,6 +445,23 @@ public final class Entity {
|
||||
: getEntityByName(ref.getType(), ref.getFullyQualifiedName(), fields, include);
|
||||
}
|
||||
|
||||
public static <T> List<T> getEntities(
|
||||
List<EntityReference> refs, String fields, Include include) {
|
||||
if (CollectionUtils.isEmpty(refs)) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
EntityRepository<?> entityRepository = Entity.getEntityRepository(refs.get(0).getType());
|
||||
@SuppressWarnings("unchecked")
|
||||
List<T> entities =
|
||||
(List<T>)
|
||||
entityRepository.get(
|
||||
null,
|
||||
refs.stream().map(EntityReference::getId).toList(),
|
||||
entityRepository.getFields(fields),
|
||||
include);
|
||||
return entities;
|
||||
}
|
||||
|
||||
public static <T> T getEntityOrNull(
|
||||
EntityReference entityReference, String field, Include include) {
|
||||
if (entityReference == null) return null;
|
||||
@ -478,6 +503,16 @@ public final class Entity {
|
||||
return getEntityByName(entityType, fqn, fields, include, true);
|
||||
}
|
||||
|
||||
public static <T> List<T> getEntityByNames(
|
||||
String entityType, List<String> tagFQNs, String fields, Include include) {
|
||||
EntityRepository<?> entityRepository = Entity.getEntityRepository(entityType);
|
||||
@SuppressWarnings("unchecked")
|
||||
List<T> entities =
|
||||
(List<T>)
|
||||
entityRepository.getByNames(null, tagFQNs, entityRepository.getFields(fields), include);
|
||||
return entities;
|
||||
}
|
||||
|
||||
/** Retrieve the corresponding entity repository for a given entity name. */
|
||||
public static EntityRepository<? extends EntityInterface> getEntityRepository(
|
||||
@NonNull String entityType) {
|
||||
|
||||
@ -876,6 +876,13 @@ public interface CollectionDAO {
|
||||
private String json;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
class EntityRelationshipCount {
|
||||
private UUID id;
|
||||
private Integer count;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
class EntityRelationshipObject {
|
||||
@ -1009,6 +1016,19 @@ public interface CollectionDAO {
|
||||
@Bind("fromEntityType") String fromEntityType,
|
||||
@Bind("toEntityType") String toEntityType);
|
||||
|
||||
@SqlQuery(
|
||||
"SELECT fromId, toId, fromEntity, toEntity, relation "
|
||||
+ "FROM entity_relationship "
|
||||
+ "WHERE fromId IN (<fromIds>) "
|
||||
+ "AND relation = :relation "
|
||||
+ "AND toEntity = :toEntityType "
|
||||
+ "AND deleted = FALSE")
|
||||
@UseRowMapper(RelationshipObjectMapper.class)
|
||||
List<EntityRelationshipObject> findToBatch(
|
||||
@BindList("fromIds") List<String> fromIds,
|
||||
@Bind("relation") int relation,
|
||||
@Bind("toEntityType") String toEntityType);
|
||||
|
||||
@SqlQuery(
|
||||
"SELECT toId, toEntity, json FROM entity_relationship "
|
||||
+ "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity")
|
||||
@ -1019,6 +1039,17 @@ public interface CollectionDAO {
|
||||
@Bind("relation") int relation,
|
||||
@Bind("toEntity") String toEntity);
|
||||
|
||||
@SqlQuery(
|
||||
"SELECT fromId, COUNT(toId) FROM entity_relationship "
|
||||
+ "WHERE fromId IN (<fromIds>) AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity "
|
||||
+ "GROUP BY fromId")
|
||||
@RegisterRowMapper(ToRelationshipCountMapper.class)
|
||||
List<EntityRelationshipCount> countFindTo(
|
||||
@BindList("fromIds") List<String> fromIds,
|
||||
@Bind("fromEntity") String fromEntity,
|
||||
@Bind("relation") int relation,
|
||||
@Bind("toEntity") String toEntity);
|
||||
|
||||
@SqlQuery(
|
||||
"SELECT COUNT(toId) FROM entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity "
|
||||
+ "AND relation IN (<relation>)")
|
||||
@ -1300,6 +1331,16 @@ public interface CollectionDAO {
|
||||
}
|
||||
}
|
||||
|
||||
class ToRelationshipCountMapper implements RowMapper<EntityRelationshipCount> {
|
||||
@Override
|
||||
public EntityRelationshipCount map(ResultSet rs, StatementContext ctx) throws SQLException {
|
||||
return EntityRelationshipCount.builder()
|
||||
.id(UUID.fromString(rs.getString(1)))
|
||||
.count(rs.getInt(2))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
class RelationshipObjectMapper implements RowMapper<EntityRelationshipObject> {
|
||||
@Override
|
||||
public EntityRelationshipObject map(ResultSet rs, StatementContext ctx) throws SQLException {
|
||||
|
||||
@ -19,11 +19,18 @@ import static org.openmetadata.service.jdbi3.ListFilter.escapeApostrophe;
|
||||
import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL;
|
||||
import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import lombok.SneakyThrows;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.jdbi.v3.core.mapper.RowMapper;
|
||||
import org.jdbi.v3.core.statement.StatementContext;
|
||||
import org.jdbi.v3.sqlobject.config.RegisterRowMapper;
|
||||
import org.jdbi.v3.sqlobject.customizer.Bind;
|
||||
import org.jdbi.v3.sqlobject.customizer.BindList;
|
||||
import org.jdbi.v3.sqlobject.customizer.BindMap;
|
||||
import org.jdbi.v3.sqlobject.customizer.Define;
|
||||
import org.jdbi.v3.sqlobject.statement.BatchChunkSize;
|
||||
@ -148,6 +155,13 @@ public interface EntityDAO<T extends EntityInterface> {
|
||||
String findById(
|
||||
@Define("table") String table, @BindUUID("id") UUID id, @Define("cond") String cond);
|
||||
|
||||
@SqlQuery("SELECT id, json FROM <table> WHERE id IN (<ids>) <cond>")
|
||||
@RegisterRowMapper(EntityIdJsonPairMapper.class)
|
||||
List<EntityIdJsonPair> findByIds(
|
||||
@Define("table") String table,
|
||||
@BindList("ids") List<String> ids,
|
||||
@Define("cond") String cond);
|
||||
|
||||
@SqlQuery("SELECT json FROM <table> WHERE <nameColumnHash> = :name <cond>")
|
||||
String findByName(
|
||||
@Define("table") String table,
|
||||
@ -155,6 +169,14 @@ public interface EntityDAO<T extends EntityInterface> {
|
||||
@BindFQN("name") String name,
|
||||
@Define("cond") String cond);
|
||||
|
||||
@SqlQuery("SELECT <nameColumnHash>, json FROM <table> WHERE <nameColumnHash> IN (<names>) <cond>")
|
||||
@RegisterRowMapper(EntityNameColumnHashJsonPairMapper.class)
|
||||
List<EntityNameColumnHashJsonPair> findByNames(
|
||||
@Define("table") String table,
|
||||
@Define("nameColumnHash") String nameColumn,
|
||||
@BindList("names") List<String> names,
|
||||
@Define("cond") String cond);
|
||||
|
||||
@SqlQuery("SELECT count(<nameHashColumn>) FROM <table> <cond>")
|
||||
int listCount(
|
||||
@Define("table") String table,
|
||||
@ -448,6 +470,19 @@ public interface EntityDAO<T extends EntityInterface> {
|
||||
return findEntityById(id, Include.NON_DELETED);
|
||||
}
|
||||
|
||||
default List<T> findEntitiesByIds(List<UUID> ids, Include include) {
|
||||
if (CollectionUtils.isEmpty(ids)) {
|
||||
return List.of();
|
||||
}
|
||||
return findByIds(
|
||||
getTableName(),
|
||||
ids.stream().map(UUID::toString).distinct().toList(),
|
||||
getCondition(include))
|
||||
.stream()
|
||||
.map(pair -> jsonToEntity(pair.json, pair.id))
|
||||
.toList();
|
||||
}
|
||||
|
||||
default T findEntityByName(String fqn) {
|
||||
return findEntityByName(fqn, Include.NON_DELETED);
|
||||
}
|
||||
@ -464,6 +499,17 @@ public interface EntityDAO<T extends EntityInterface> {
|
||||
findByName(getTableName(), nameHashColumn, fqn, getCondition(include)), fqn);
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
default List<T> findEntityByNames(List<String> entityFQNs, Include include) {
|
||||
if (CollectionUtils.isEmpty(entityFQNs)) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> names = entityFQNs.stream().distinct().map(FullyQualifiedName::buildHash).toList();
|
||||
return findByNames(getTableName(), getNameHashColumn(), names, getCondition(include)).stream()
|
||||
.map(pair -> jsonToEntity(pair.json, pair.nameColumnHash))
|
||||
.toList();
|
||||
}
|
||||
|
||||
default T jsonToEntity(String json, Object identity) {
|
||||
Class<T> clz = getEntityClass();
|
||||
T entity = json != null ? JsonUtils.readValue(json, clz) : null;
|
||||
@ -549,4 +595,22 @@ public interface EntityDAO<T extends EntityInterface> {
|
||||
throw EntityNotFoundException.byMessage(entityNotFound(entityType, id));
|
||||
}
|
||||
}
|
||||
|
||||
record EntityNameColumnHashJsonPair(String nameColumnHash, String json) {}
|
||||
|
||||
class EntityNameColumnHashJsonPairMapper implements RowMapper<EntityNameColumnHashJsonPair> {
|
||||
@Override
|
||||
public EntityNameColumnHashJsonPair map(ResultSet r, StatementContext ctx) throws SQLException {
|
||||
return new EntityNameColumnHashJsonPair(r.getString(1), r.getString(2));
|
||||
}
|
||||
}
|
||||
|
||||
record EntityIdJsonPair(UUID id, String json) {}
|
||||
|
||||
class EntityIdJsonPairMapper implements RowMapper<EntityIdJsonPair> {
|
||||
@Override
|
||||
public EntityIdJsonPair map(ResultSet r, StatementContext ctx) throws SQLException {
|
||||
return new EntityIdJsonPair(UUID.fromString(r.getString(1)), r.getString(2));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -120,6 +120,7 @@ import javax.ws.rs.core.UriInfo;
|
||||
import lombok.Getter;
|
||||
import lombok.NonNull;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.tuple.ImmutablePair;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.jdbi.v3.sqlobject.transaction.Transaction;
|
||||
@ -132,6 +133,7 @@ import org.openmetadata.schema.api.VoteRequest.VoteType;
|
||||
import org.openmetadata.schema.api.feed.ResolveTask;
|
||||
import org.openmetadata.schema.api.teams.CreateTeam;
|
||||
import org.openmetadata.schema.configuration.AssetCertificationSettings;
|
||||
import org.openmetadata.schema.entity.classification.Tag;
|
||||
import org.openmetadata.schema.entity.data.Table;
|
||||
import org.openmetadata.schema.entity.feed.Suggestion;
|
||||
import org.openmetadata.schema.entity.teams.Team;
|
||||
@ -257,6 +259,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
@Getter protected final boolean supportsStyle;
|
||||
@Getter protected final boolean supportsLifeCycle;
|
||||
@Getter protected final boolean supportsCertification;
|
||||
@Getter protected final boolean supportsChildren;
|
||||
protected final boolean supportsFollower;
|
||||
protected final boolean supportsExtension;
|
||||
protected final boolean supportsVotes;
|
||||
@ -373,6 +376,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
this.patchFields.addField(allowedFields, FIELD_CERTIFICATION);
|
||||
this.putFields.addField(allowedFields, FIELD_CERTIFICATION);
|
||||
}
|
||||
this.supportsChildren = allowedFields.contains(FIELD_CHILDREN);
|
||||
|
||||
Map<String, Pair<Boolean, BiConsumer<List<T>, Fields>>> fieldSupportMap = new HashMap<>();
|
||||
|
||||
@ -381,6 +385,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
fieldSupportMap.put(FIELD_DOMAIN, Pair.of(supportsDomain, this::fetchAndSetDomain));
|
||||
fieldSupportMap.put(FIELD_REVIEWERS, Pair.of(supportsReviewers, this::fetchAndSetReviewers));
|
||||
fieldSupportMap.put(FIELD_EXTENSION, Pair.of(supportsExtension, this::fetchAndSetExtension));
|
||||
fieldSupportMap.put(FIELD_CHILDREN, Pair.of(supportsChildren, this::fetchAndSetChildren));
|
||||
|
||||
for (Entry<String, Pair<Boolean, BiConsumer<List<T>, Fields>>> entry :
|
||||
fieldSupportMap.entrySet()) {
|
||||
@ -390,8 +395,6 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
|
||||
if (supportsField) {
|
||||
this.fieldFetchers.put(fieldName, fetcher);
|
||||
this.patchFields.addField(allowedFields, fieldName);
|
||||
this.putFields.addField(allowedFields, fieldName);
|
||||
}
|
||||
}
|
||||
|
||||
@ -474,7 +477,11 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
/**
|
||||
* The default behavior is to execute one by one. For batch execution, override this method in the subclass.
|
||||
*
|
||||
* @see GlossaryTermRepository#setInheritedFields(List, Fields) for an example implementation
|
||||
*/
|
||||
protected void setInheritedFields(List<T> entities, Fields fields) {
|
||||
for (T entity : entities) {
|
||||
setInheritedFields(entity, fields);
|
||||
@ -633,6 +640,13 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
return withHref(uriInfo, entityClone);
|
||||
}
|
||||
|
||||
public final List<T> get(UriInfo uriInfo, List<UUID> ids, Fields fields, Include include) {
|
||||
List<T> entities = find(ids, include);
|
||||
setFieldsInBulk(fields, entities);
|
||||
entities.forEach(entity -> withHref(uriInfo, entity));
|
||||
return entities;
|
||||
}
|
||||
|
||||
/**
|
||||
* getReference is used for getting the entity references from the entity in the cache.
|
||||
*/
|
||||
@ -641,6 +655,11 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
return find(id, include).getEntityReference();
|
||||
}
|
||||
|
||||
public final List<EntityReference> getReferences(List<UUID> id, Include include)
|
||||
throws EntityNotFoundException {
|
||||
return find(id, include).stream().map(EntityInterface::getEntityReference).toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Find method is used for getting an entity only with core fields stored as JSON without any relational fields set
|
||||
*/
|
||||
@ -665,6 +684,10 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
}
|
||||
}
|
||||
|
||||
public final List<T> find(List<UUID> ids, Include include) {
|
||||
return dao.findEntitiesByIds(ids, include);
|
||||
}
|
||||
|
||||
public T getByName(UriInfo uriInfo, String fqn, Fields fields) {
|
||||
return getByName(uriInfo, fqn, fields, NON_DELETED, false);
|
||||
}
|
||||
@ -695,6 +718,14 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
return findByName(fqn, include).getEntityReference();
|
||||
}
|
||||
|
||||
public final List<T> getByNames(
|
||||
UriInfo uriInfo, List<String> entityFQNs, Fields fields, Include include) {
|
||||
List<T> entities = findByNames(entityFQNs, include);
|
||||
setFieldsInBulk(fields, entities);
|
||||
entities.forEach(entity -> withHref(uriInfo, entity));
|
||||
return entities;
|
||||
}
|
||||
|
||||
public final T findByNameOrNull(String fqn, Include include) {
|
||||
try {
|
||||
return findByName(fqn, include);
|
||||
@ -728,16 +759,19 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
}
|
||||
}
|
||||
|
||||
public List<T> findByNames(List<String> entityFQNs, Include include) {
|
||||
return dao.findEntityByNames(entityFQNs, include);
|
||||
}
|
||||
|
||||
public final List<T> listAll(Fields fields, ListFilter filter) {
|
||||
// forward scrolling, if after == null then first page is being asked
|
||||
List<String> jsons = dao.listAfter(filter, Integer.MAX_VALUE, "", "");
|
||||
List<T> entities = new ArrayList<>();
|
||||
for (String json : jsons) {
|
||||
T entity = setFieldsInternal(JsonUtils.readValue(json, entityClass), fields);
|
||||
setInheritedFields(entity, fields);
|
||||
clearFieldsInternal(entity, fields);
|
||||
T entity = JsonUtils.readValue(json, entityClass);
|
||||
entities.add(entity);
|
||||
}
|
||||
setFieldsInBulk(fields, entities);
|
||||
return entities;
|
||||
}
|
||||
|
||||
@ -745,17 +779,32 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
ListFilter filter = new ListFilter(NON_DELETED);
|
||||
List<String> jsons = listAllByParentFqn(parentFqn, filter);
|
||||
List<T> entities = new ArrayList<>();
|
||||
setFieldsInBulk(jsons, fields, entities);
|
||||
return entities;
|
||||
}
|
||||
|
||||
public void setFieldsInBulk(List<String> jsons, Fields fields, List<T> entities) {
|
||||
for (String json : jsons) {
|
||||
T entity = JsonUtils.readValue(json, entityClass);
|
||||
entities.add(entity);
|
||||
}
|
||||
// TODO: Ensure consistent behavior with setFieldsInBulk when all repositories implement it
|
||||
fetchAndSetFields(entities, fields);
|
||||
setInheritedFields(entities, fields);
|
||||
for (T entity : entities) {
|
||||
clearFieldsInternal(entity, fields);
|
||||
}
|
||||
return entities;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes {@link #setFields} on a list of entities. By default, this method processes
|
||||
* each entity individually. To enable batch processing, override this method in a subclass.
|
||||
* <p>
|
||||
* For efficient bulk processing, ensure all fields used in {@link #setFields}
|
||||
* have corresponding batch processing methods, such as {@code fetchAndSetXXX}. For instance,
|
||||
* if handling a domain field, implement {@link #fetchAndSetDomain}.
|
||||
* <p>
|
||||
* Example implementation can be found in {@link GlossaryTermRepository#setFieldsInBulk}.
|
||||
*/
|
||||
public void setFieldsInBulk(Fields fields, List<T> entities) {
|
||||
for (T entity : entities) {
|
||||
setFieldsInternal(entity, fields);
|
||||
setInheritedFields(entity, fields);
|
||||
clearFieldsInternal(entity, fields);
|
||||
}
|
||||
@ -788,11 +837,11 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
List<String> jsons = dao.listAfter(filter, limitParam + 1, afterName, afterId);
|
||||
|
||||
for (String json : jsons) {
|
||||
T entity = setFieldsInternal(JsonUtils.readValue(json, entityClass), fields);
|
||||
setInheritedFields(entity, fields);
|
||||
clearFieldsInternal(entity, fields);
|
||||
entities.add(withHref(uriInfo, entity));
|
||||
T entity = JsonUtils.readValue(json, entityClass);
|
||||
entities.add(entity);
|
||||
}
|
||||
setFieldsInBulk(fields, entities);
|
||||
entities.forEach(entity -> withHref(uriInfo, entity));
|
||||
|
||||
String beforeCursor;
|
||||
String afterCursor = null;
|
||||
@ -834,11 +883,12 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
|
||||
List<T> entities = new ArrayList<>();
|
||||
for (String json : jsons) {
|
||||
T entity = setFieldsInternal(JsonUtils.readValue(json, entityClass), fields);
|
||||
setInheritedFields(entity, fields);
|
||||
clearFieldsInternal(entity, fields);
|
||||
entities.add(withHref(uriInfo, entity));
|
||||
T entity = JsonUtils.readValue(json, entityClass);
|
||||
entities.add(entity);
|
||||
}
|
||||
setFieldsInBulk(fields, entities);
|
||||
entities.forEach(entity -> withHref(uriInfo, entity));
|
||||
|
||||
int total = dao.listCount(filter);
|
||||
|
||||
String beforeCursor = null;
|
||||
@ -3943,13 +3993,8 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
}
|
||||
}
|
||||
|
||||
private void fetchAndSetFields(List<T> entities, Fields fields) {
|
||||
for (String field : fields) {
|
||||
BiConsumer<List<T>, Fields> fetcher = fieldFetchers.get(field);
|
||||
if (fetcher != null) {
|
||||
fetcher.accept(entities, fields);
|
||||
}
|
||||
}
|
||||
protected void fetchAndSetFields(List<T> entities, Fields fields) {
|
||||
fieldFetchers.values().forEach(fetcher -> fetcher.accept(entities, fields));
|
||||
}
|
||||
|
||||
private void fetchAndSetOwners(List<T> entities, Fields fields) {
|
||||
@ -4008,6 +4053,18 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
}
|
||||
}
|
||||
|
||||
protected void fetchAndSetChildren(List<T> entities, Fields fields) {
|
||||
if (!fields.contains(FIELD_CHILDREN) || entities == null || entities.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Map<UUID, List<EntityReference>> childrenMap = batchFetchChildren(entities);
|
||||
|
||||
for (T entity : entities) {
|
||||
entity.setChildren(childrenMap.get(entity.getId()));
|
||||
}
|
||||
}
|
||||
|
||||
private void fetchAndSetReviewers(List<T> entities, Fields fields) {
|
||||
if (!fields.contains(FIELD_REVIEWERS) || !supportsReviewers || entities.isEmpty()) {
|
||||
return;
|
||||
@ -4051,6 +4108,15 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
List<CollectionDAO.TagUsageDAO.TagLabelWithFQNHash> tags =
|
||||
daoCollection.tagUsageDAO().getTagsInternalBatch(entityFQNs);
|
||||
|
||||
List<String> tagFQNs =
|
||||
tags.stream()
|
||||
.distinct()
|
||||
.map(CollectionDAO.TagUsageDAO.TagLabelWithFQNHash::getTagFQN)
|
||||
.toList();
|
||||
Map<String, TagLabel> tagFQNLabelMap =
|
||||
EntityUtil.toTagLabels(TagLabelUtil.getTags(tagFQNs).toArray(new Tag[0])).stream()
|
||||
.collect(Collectors.toMap(TagLabel::getTagFQN, Function.identity()));
|
||||
|
||||
// Map from targetFQNHash to targetFQN
|
||||
Map<String, String> fqnHashToFqnMap =
|
||||
entityFQNs.stream().collect(Collectors.toMap(FullyQualifiedName::buildHash, fqn -> fqn));
|
||||
@ -4062,7 +4128,8 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
String targetFQN = fqnHashToFqnMap.get(targetFQNHash);
|
||||
|
||||
if (targetFQN != null) {
|
||||
tagsMap.computeIfAbsent(targetFQN, k -> new ArrayList<>()).add(tagWithHash.toTagLabel());
|
||||
TagLabel tagLabel = tagFQNLabelMap.get(tagWithHash.getTagFQN());
|
||||
tagsMap.computeIfAbsent(targetFQN, k -> new ArrayList<>()).add(tagLabel);
|
||||
}
|
||||
}
|
||||
|
||||
@ -4152,6 +4219,36 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
return result;
|
||||
}
|
||||
|
||||
private Map<UUID, List<EntityReference>> batchFetchChildren(List<T> entities) {
|
||||
List<CollectionDAO.EntityRelationshipObject> records =
|
||||
daoCollection
|
||||
.relationshipDAO()
|
||||
.findToBatch(
|
||||
entityListToStrings(entities), Relationship.CONTAINS.ordinal(), entityType);
|
||||
|
||||
Map<UUID, List<EntityReference>> childrenMap = new HashMap<>();
|
||||
|
||||
if (CollectionUtils.isEmpty(records)) {
|
||||
return childrenMap;
|
||||
}
|
||||
|
||||
Map<String, EntityReference> idReferenceMap =
|
||||
Entity.getEntityReferencesByIds(
|
||||
records.get(0).getToEntity(),
|
||||
records.stream().map(e -> UUID.fromString(e.getToId())).distinct().toList(),
|
||||
ALL)
|
||||
.stream()
|
||||
.collect(Collectors.toMap(e -> e.getId().toString(), Function.identity()));
|
||||
|
||||
for (CollectionDAO.EntityRelationshipObject rec : records) {
|
||||
UUID entityId = UUID.fromString(rec.getFromId());
|
||||
EntityReference childrenRef = idReferenceMap.get(rec.getToId());
|
||||
childrenMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(childrenRef);
|
||||
}
|
||||
|
||||
return childrenMap;
|
||||
}
|
||||
|
||||
private List<String> entityListToStrings(List<T> entities) {
|
||||
return entities.stream().map(EntityInterface::getId).map(UUID::toString).toList();
|
||||
}
|
||||
|
||||
@ -41,6 +41,7 @@ import static org.openmetadata.service.util.EntityUtil.termReferenceMatch;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.google.gson.Gson;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
@ -57,6 +58,7 @@ import java.util.stream.Collectors;
|
||||
import javax.json.JsonPatch;
|
||||
import javax.ws.rs.core.Response;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.tuple.ImmutablePair;
|
||||
import org.jdbi.v3.sqlobject.transaction.Transaction;
|
||||
import org.openmetadata.common.utils.CommonUtil;
|
||||
@ -121,8 +123,10 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
|
||||
UPDATE_FIELDS);
|
||||
supportsSearch = true;
|
||||
renameAllowed = true;
|
||||
fieldFetchers.put("relatedTerms", this::fetchAndSetRelatedTerms);
|
||||
fieldFetchers.put("parent", this::fetchAndSetParentOrGlossary);
|
||||
fieldFetchers.put("relatedTerms", this::fetchAndSetRelatedTerms);
|
||||
fieldFetchers.put("usageCount", this::fetchAndSetUsageCount);
|
||||
fieldFetchers.put("childrenCount", this::fetchAndSetChildrenCount);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -136,6 +140,15 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
|
||||
fields.contains("childrenCount") ? getChildrenCount(entity) : entity.getChildrenCount());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setFieldsInBulk(Fields fields, List<GlossaryTerm> entities) {
|
||||
fetchAndSetFields(entities, fields);
|
||||
setInheritedFields(entities, fields);
|
||||
for (GlossaryTerm entity : entities) {
|
||||
clearFieldsInternal(entity, fields);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clearFields(GlossaryTerm entity, Fields fields) {
|
||||
entity.setRelatedTerms(fields.contains("relatedTerms") ? entity.getRelatedTerms() : null);
|
||||
@ -151,6 +164,25 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
|
||||
inheritReviewers(glossaryTerm, fields, parent);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setInheritedFields(List<GlossaryTerm> glossaryTerms, Fields fields) {
|
||||
List<? extends EntityInterface> parents =
|
||||
getParentEntities(glossaryTerms, "owners,domain,reviewers");
|
||||
Map<UUID, EntityInterface> parentMap =
|
||||
parents.stream().collect(Collectors.toMap(EntityInterface::getId, e -> e));
|
||||
for (GlossaryTerm glossaryTerm : glossaryTerms) {
|
||||
EntityInterface parent = null;
|
||||
if (glossaryTerm.getParent() != null) {
|
||||
parent = parentMap.get(glossaryTerm.getParent().getId());
|
||||
} else if (glossaryTerm.getGlossary() != null) {
|
||||
parent = parentMap.get(glossaryTerm.getGlossary().getId());
|
||||
}
|
||||
inheritOwners(glossaryTerm, fields, parent);
|
||||
inheritDomain(glossaryTerm, fields, parent);
|
||||
inheritReviewers(glossaryTerm, fields, parent);
|
||||
}
|
||||
}
|
||||
|
||||
private Integer getUsageCount(GlossaryTerm term) {
|
||||
return daoCollection
|
||||
.tagUsageDAO()
|
||||
@ -218,6 +250,24 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
|
||||
.withReviewers(reviewers);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void storeEntities(List<GlossaryTerm> entities) {
|
||||
List<GlossaryTerm> entitiesToStore = new ArrayList<>();
|
||||
Gson gson = new Gson();
|
||||
for (GlossaryTerm entity : entities) {
|
||||
EntityReference glossary = entity.getGlossary();
|
||||
EntityReference parentTerm = entity.getParent();
|
||||
List<EntityReference> reviewers = entity.getReviewers();
|
||||
|
||||
String jsonCopy = gson.toJson(entity.withGlossary(null).withParent(null).withReviewers(null));
|
||||
entitiesToStore.add(gson.fromJson(jsonCopy, GlossaryTerm.class));
|
||||
|
||||
// restore the relationships
|
||||
entity.withGlossary(glossary).withParent(parentTerm).withReviewers(reviewers);
|
||||
}
|
||||
storeMany(entitiesToStore);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void storeRelationships(GlossaryTerm entity) {
|
||||
addGlossaryRelationship(entity);
|
||||
@ -625,6 +675,24 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
|
||||
: Entity.getEntity(entity.getGlossary(), fields, Include.ALL);
|
||||
}
|
||||
|
||||
public List<EntityInterface> getParentEntities(List<GlossaryTerm> entities, String fields) {
|
||||
List<EntityInterface> result = new ArrayList<>();
|
||||
if (CollectionUtils.isEmpty(entities)) {
|
||||
return result;
|
||||
}
|
||||
List<EntityReference> parents =
|
||||
entities.stream().map(GlossaryTerm::getParent).filter(Objects::nonNull).distinct().toList();
|
||||
result.addAll(Entity.getEntities(parents, fields, Include.ALL));
|
||||
List<EntityReference> glossaries =
|
||||
entities.stream()
|
||||
.map(GlossaryTerm::getGlossary)
|
||||
.filter(Objects::nonNull)
|
||||
.distinct()
|
||||
.toList();
|
||||
result.addAll(Entity.getEntities(glossaries, fields, Include.ALL));
|
||||
return result;
|
||||
}
|
||||
|
||||
private void addGlossaryRelationship(GlossaryTerm term) {
|
||||
Relationship relationship = term.getParent() != null ? Relationship.HAS : Relationship.CONTAINS;
|
||||
addRelationship(
|
||||
@ -841,11 +909,12 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
|
||||
}
|
||||
|
||||
private void fetchAndSetParentOrGlossary(List<GlossaryTerm> terms, Fields fields) {
|
||||
if (terms == null || terms.isEmpty() || (!fields.contains("parent"))) {
|
||||
if (terms == null || terms.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
List<String> entityIds = terms.stream().map(GlossaryTerm::getId).map(UUID::toString).toList();
|
||||
List<String> entityIds =
|
||||
terms.stream().map(GlossaryTerm::getId).map(UUID::toString).distinct().toList();
|
||||
|
||||
List<CollectionDAO.EntityRelationshipObject> parentRecords =
|
||||
daoCollection
|
||||
@ -919,6 +988,39 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
|
||||
}
|
||||
}
|
||||
|
||||
private void fetchAndSetUsageCount(List<GlossaryTerm> entities, Fields fields) {
|
||||
if (!fields.contains("usageCount") || entities.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
// TODO: modify to use a single db query
|
||||
for (GlossaryTerm entity : entities) {
|
||||
entity.withUsageCount(getUsageCount(entity));
|
||||
}
|
||||
}
|
||||
|
||||
private void fetchAndSetChildrenCount(List<GlossaryTerm> entities, Fields fields) {
|
||||
if (!fields.contains("childrenCount") || entities.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
List<String> termIds =
|
||||
entities.stream().map(GlossaryTerm::getId).map(UUID::toString).distinct().toList();
|
||||
|
||||
Map<UUID, Integer> termIdCountMap =
|
||||
daoCollection
|
||||
.relationshipDAO()
|
||||
.countFindTo(termIds, GLOSSARY_TERM, Relationship.CONTAINS.ordinal(), GLOSSARY_TERM)
|
||||
.stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
CollectionDAO.EntityRelationshipCount::getId,
|
||||
CollectionDAO.EntityRelationshipCount::getCount));
|
||||
|
||||
for (GlossaryTerm entity : entities) {
|
||||
entity.setChildrenCount(
|
||||
termIdCountMap.getOrDefault(entity.getId(), entity.getChildrenCount()));
|
||||
}
|
||||
}
|
||||
|
||||
/** Handles entity updated from PUT and POST operation. */
|
||||
public class GlossaryTermUpdater extends EntityUpdater {
|
||||
public GlossaryTermUpdater(GlossaryTerm original, GlossaryTerm updated, Operation operation) {
|
||||
|
||||
@ -416,6 +416,36 @@ public class GlossaryTermResource extends EntityResource<GlossaryTerm, GlossaryT
|
||||
return create(uriInfo, securityContext, term);
|
||||
}
|
||||
|
||||
@POST
|
||||
@Path("/createMany")
|
||||
@Operation(
|
||||
operationId = "createManyGlossaryTerm",
|
||||
summary = "Create multiple glossary terms at once",
|
||||
description = "Create multiple new glossary terms.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "The glossary term",
|
||||
content =
|
||||
@Content(
|
||||
mediaType = "application/json",
|
||||
schema = @Schema(implementation = GlossaryTerm.class))),
|
||||
@ApiResponse(responseCode = "400", description = "Bad request")
|
||||
})
|
||||
public Response createMany(
|
||||
@Context UriInfo uriInfo,
|
||||
@Context SecurityContext securityContext,
|
||||
@Valid List<CreateGlossaryTerm> creates) {
|
||||
List<GlossaryTerm> terms =
|
||||
creates.stream()
|
||||
.map(
|
||||
create ->
|
||||
mapper.createToEntity(create, securityContext.getUserPrincipal().getName()))
|
||||
.toList();
|
||||
List<GlossaryTerm> result = repository.createMany(uriInfo, terms);
|
||||
return Response.ok(result).build();
|
||||
}
|
||||
|
||||
@PATCH
|
||||
@Path("/{id}")
|
||||
@Operation(
|
||||
|
||||
@ -52,6 +52,10 @@ public class TagLabelUtil {
|
||||
return Entity.getEntityByName(Entity.TAG, tagFqn, "", NON_DELETED);
|
||||
}
|
||||
|
||||
public static List<Tag> getTags(List<String> tagFQNs) {
|
||||
return Entity.getEntityByNames(Entity.TAG, tagFQNs, "", NON_DELETED);
|
||||
}
|
||||
|
||||
public static Glossary getGlossary(String glossaryName) {
|
||||
return Entity.getEntityByName(Entity.GLOSSARY, glossaryName, "", NON_DELETED);
|
||||
}
|
||||
|
||||
@ -20,6 +20,7 @@ import static java.util.Collections.emptyList;
|
||||
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
|
||||
import static javax.ws.rs.core.Response.Status.FORBIDDEN;
|
||||
import static javax.ws.rs.core.Response.Status.NOT_FOUND;
|
||||
import static javax.ws.rs.core.Response.Status.OK;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.openmetadata.common.utils.CommonUtil.listOf;
|
||||
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
|
||||
@ -56,6 +57,8 @@ import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.IntStream;
|
||||
import javax.ws.rs.client.WebTarget;
|
||||
import javax.ws.rs.core.Response;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.HttpResponseException;
|
||||
@ -1057,6 +1060,21 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_performance_listEntities() throws IOException {
|
||||
Glossary glossary = createGlossary("词汇表三", null, null);
|
||||
List<Map<String, Object>> result =
|
||||
createTerms(glossary, IntStream.range(0, 500).mapToObj(i -> "term" + i).toList());
|
||||
|
||||
Map<String, String> queryParams = new HashMap<>();
|
||||
queryParams.put("fields", "children,relatedTerms,reviewers,tags");
|
||||
queryParams.put("limit", "10000");
|
||||
queryParams.put("directChildrenOf", "词汇表三");
|
||||
ResultList<GlossaryTerm> list =
|
||||
assertTimeout(Duration.ofSeconds(3), () -> listEntities(queryParams, ADMIN_AUTH_HEADERS));
|
||||
assertEquals(result.size(), list.getData().size());
|
||||
}
|
||||
|
||||
public GlossaryTerm createTerm(Glossary glossary, GlossaryTerm parent, String termName)
|
||||
throws IOException {
|
||||
return createTerm(glossary, parent, termName, glossary.getReviewers());
|
||||
@ -1086,6 +1104,25 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
|
||||
return createAndCheckEntity(createGlossaryTerm, createdBy);
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> createTerms(Glossary glossary, List<String> termNames)
|
||||
throws HttpResponseException {
|
||||
String pathUrl = "/createMany/";
|
||||
String glossaryFqn = getFqn(glossary);
|
||||
WebTarget target = getCollection().path(pathUrl);
|
||||
List<CreateGlossaryTerm> createGlossaryTerms =
|
||||
termNames.stream()
|
||||
.map(
|
||||
name ->
|
||||
createRequest(name, "d", "", null)
|
||||
.withRelatedTerms(null)
|
||||
.withSynonyms(List.of("performance1", "performance2"))
|
||||
.withStyle(new Style().withColor("#FF5733").withIconURL("https://img"))
|
||||
.withGlossary(glossaryFqn))
|
||||
.toList();
|
||||
return TestUtils.post(
|
||||
target, createGlossaryTerms, List.class, OK.getStatusCode(), ADMIN_AUTH_HEADERS);
|
||||
}
|
||||
|
||||
public void assertContains(List<GlossaryTerm> expectedTerms, List<GlossaryTerm> actualTerms)
|
||||
throws HttpResponseException {
|
||||
assertEquals(expectedTerms.size(), actualTerms.size());
|
||||
@ -1341,6 +1378,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
|
||||
// List children glossary terms with term1 as the parent and getting immediate children only
|
||||
Map<String, String> queryParams = new HashMap<>();
|
||||
queryParams.put("directChildrenOf", term1.getFullyQualifiedName());
|
||||
queryParams.put("fields", "childrenCount,children");
|
||||
List<GlossaryTerm> children = listEntities(queryParams, ADMIN_AUTH_HEADERS).getData();
|
||||
|
||||
assertEquals(term1.getChildren().size(), children.size());
|
||||
@ -1348,10 +1386,21 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
|
||||
for (GlossaryTerm responseChild : children) {
|
||||
assertTrue(
|
||||
responseChild.getFullyQualifiedName().startsWith(responseChild.getFullyQualifiedName()));
|
||||
if (responseChild.getChildren() == null) {
|
||||
assertNull(responseChild.getChildrenCount());
|
||||
} else {
|
||||
assertEquals(responseChild.getChildren().size(), responseChild.getChildrenCount());
|
||||
}
|
||||
}
|
||||
|
||||
GlossaryTerm response = getEntity(term1.getId(), "childrenCount", ADMIN_AUTH_HEADERS);
|
||||
assertEquals(term1.getChildren().size(), response.getChildrenCount());
|
||||
|
||||
queryParams = new HashMap<>();
|
||||
queryParams.put("directChildrenOf", glossary1.getFullyQualifiedName());
|
||||
queryParams.put("fields", "childrenCount");
|
||||
children = listEntities(queryParams, ADMIN_AUTH_HEADERS).getData();
|
||||
assertEquals(term1.getChildren().size(), children.get(0).getChildrenCount());
|
||||
}
|
||||
|
||||
public Glossary createGlossary(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user