Fixes #3744 Clean up EntityUtil (#3745)

This commit is contained in:
Suresh Srinivas 2022-03-29 16:30:27 -07:00 committed by GitHub
parent 40fa4dcdcb
commit fc64c953b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 190 additions and 306 deletions

View File

@ -179,7 +179,7 @@ public final class Entity {
public static EntityReference getOwner(@NonNull EntityReference reference) throws IOException { public static EntityReference getOwner(@NonNull EntityReference reference) throws IOException {
EntityRepository<?> repository = getEntityRepository(reference.getType()); EntityRepository<?> repository = getEntityRepository(reference.getType());
return repository.getOwner(reference.getId(), reference.getType()); return repository.getOwner(reference);
} }
public static void withHref(UriInfo uriInfo, List<EntityReference> list) { public static void withHref(UriInfo uriInfo, List<EntityReference> list) {

View File

@ -18,7 +18,6 @@ import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.util.UUID; import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.operations.pipelines.AirflowPipeline; import org.openmetadata.catalog.operations.pipelines.AirflowPipeline;
import org.openmetadata.catalog.resources.operations.AirflowPipelineResource; import org.openmetadata.catalog.resources.operations.AirflowPipelineResource;
@ -50,11 +49,6 @@ public class AirflowPipelineRepository extends EntityRepository<AirflowPipeline>
: null; : null;
} }
@Transaction
public EntityReference getOwnerReference(AirflowPipeline airflowPipeline) throws IOException {
return EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), airflowPipeline.getOwner());
}
@Override @Override
public AirflowPipeline setFields(AirflowPipeline airflowPipeline, Fields fields) throws IOException { public AirflowPipeline setFields(AirflowPipeline airflowPipeline, Fields fields) throws IOException {
airflowPipeline.setService(getService(airflowPipeline)); airflowPipeline.setService(getService(airflowPipeline));

View File

@ -22,7 +22,6 @@ import java.net.URI;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Dashboard; import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.entity.services.DashboardService; import org.openmetadata.catalog.entity.services.DashboardService;
@ -63,11 +62,6 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
return new DashboardEntityInterface(entity); return new DashboardEntityInterface(entity);
} }
@Transaction
public EntityReference getOwnerReference(Dashboard dashboard) throws IOException {
return EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), dashboard.getOwner());
}
@Override @Override
public Dashboard setFields(Dashboard dashboard, Fields fields) throws IOException { public Dashboard setFields(Dashboard dashboard, Fields fields) throws IOException {
dashboard.setDisplayName(dashboard.getDisplayName()); dashboard.setDisplayName(dashboard.getDisplayName());
@ -124,7 +118,7 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
public void prepare(Dashboard dashboard) throws IOException { public void prepare(Dashboard dashboard) throws IOException {
populateService(dashboard); populateService(dashboard);
dashboard.setFullyQualifiedName(getFQN(dashboard)); dashboard.setFullyQualifiedName(getFQN(dashboard));
EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), dashboard.getOwner()); // Validate owner populateOwner(dashboard.getOwner()); // Validate owner
dashboard.setTags(addDerivedTags(dashboard.getTags())); dashboard.setTags(addDerivedTags(dashboard.getTags()));
dashboard.setCharts(getCharts(dashboard.getCharts())); dashboard.setCharts(getCharts(dashboard.getCharts()));
} }
@ -156,7 +150,7 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
} }
} }
// Add owner relationship // Add owner relationship
setOwner(dashboard.getId(), Entity.DASHBOARD, dashboard.getOwner()); addOwnerRelationship(dashboard.getId(), Entity.DASHBOARD, dashboard.getOwner());
// Add tag to dashboard relationship // Add tag to dashboard relationship
applyTags(dashboard); applyTags(dashboard);

View File

@ -13,15 +13,6 @@
package org.openmetadata.catalog.jdbi3; package org.openmetadata.catalog.jdbi3;
import static javax.ws.rs.core.Response.Status.CREATED;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.catalog.type.Include.ALL;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.UUID;
import javax.ws.rs.core.Response.Status;
import org.jdbi.v3.sqlobject.transaction.Transaction; import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Database; import org.openmetadata.catalog.entity.data.Database;
@ -35,7 +26,16 @@ import org.openmetadata.catalog.type.Relationship;
import org.openmetadata.catalog.util.EntityInterface; import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.JsonUtils;
import javax.ws.rs.core.Response.Status;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.UUID;
import static javax.ws.rs.core.Response.Status.CREATED;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.catalog.type.Include.ALL;
public class DatabaseRepository extends EntityRepository<Database> { public class DatabaseRepository extends EntityRepository<Database> {
private static final String DATABASE_UPDATE_FIELDS = "owner"; private static final String DATABASE_UPDATE_FIELDS = "owner";
@ -67,9 +67,7 @@ public class DatabaseRepository extends EntityRepository<Database> {
public void prepare(Database database) throws IOException { public void prepare(Database database) throws IOException {
populateService(database); populateService(database);
database.setFullyQualifiedName(getFQN(database)); database.setFullyQualifiedName(getFQN(database));
database.setOwner( populateOwner(database.getOwner()); // Validate owner
EntityUtil.populateOwner(
daoCollection.userDAO(), daoCollection.teamDAO(), database.getOwner())); // Validate owner
} }
@Override @Override
@ -81,11 +79,7 @@ public class DatabaseRepository extends EntityRepository<Database> {
// Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships // Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships
database.withOwner(null).withService(null).withHref(null); database.withOwner(null).withService(null).withHref(null);
if (update) { store(database.getId(), database, update);
daoCollection.databaseDAO().update(database.getId(), JsonUtils.pojoToJson(database));
} else {
daoCollection.databaseDAO().insert(database);
}
// Restore the relationships // Restore the relationships
database.withOwner(owner).withService(service); database.withOwner(owner).withService(service);
@ -95,7 +89,7 @@ public class DatabaseRepository extends EntityRepository<Database> {
public void storeRelationships(Database database) { public void storeRelationships(Database database) {
EntityReference service = database.getService(); EntityReference service = database.getService();
addRelationship(service.getId(), database.getId(), service.getType(), Entity.DATABASE, Relationship.CONTAINS); addRelationship(service.getId(), database.getId(), service.getType(), Entity.DATABASE, Relationship.CONTAINS);
setOwner(database.getId(), Entity.DATABASE, database.getOwner()); addOwnerRelationship(database.getId(), Entity.DATABASE, database.getOwner());
} }
private List<EntityReference> getTables(Database database) throws IOException { private List<EntityReference> getTables(Database database) throws IOException {

View File

@ -13,31 +13,7 @@
package org.openmetadata.catalog.jdbi3; package org.openmetadata.catalog.jdbi3;
import static org.openmetadata.catalog.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.catalog.Entity.getEntityFields;
import static org.openmetadata.catalog.type.Include.ALL;
import static org.openmetadata.catalog.type.Include.DELETED;
import static org.openmetadata.catalog.util.EntityUtil.compareTagLabel;
import static org.openmetadata.catalog.util.EntityUtil.entityReferenceMatch;
import static org.openmetadata.catalog.util.EntityUtil.nextMajorVersion;
import static org.openmetadata.catalog.util.EntityUtil.nextVersion;
import static org.openmetadata.catalog.util.EntityUtil.objectMatch;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.function.BiPredicate;
import java.util.regex.Pattern;
import javax.json.JsonPatch;
import javax.ws.rs.core.Response.Status;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.maven.shared.utils.io.IOUtil; import org.apache.maven.shared.utils.io.IOUtil;
import org.jdbi.v3.sqlobject.transaction.Transaction; import org.jdbi.v3.sqlobject.transaction.Transaction;
@ -73,6 +49,31 @@ import org.openmetadata.catalog.util.RestUtil.PutResponse;
import org.openmetadata.catalog.util.ResultList; import org.openmetadata.catalog.util.ResultList;
import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.common.utils.CommonUtil;
import javax.json.JsonPatch;
import javax.ws.rs.core.Response.Status;
import javax.ws.rs.core.UriInfo;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.function.BiPredicate;
import java.util.regex.Pattern;
import static org.openmetadata.catalog.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.catalog.Entity.getEntityFields;
import static org.openmetadata.catalog.type.Include.ALL;
import static org.openmetadata.catalog.type.Include.DELETED;
import static org.openmetadata.catalog.util.EntityUtil.compareTagLabel;
import static org.openmetadata.catalog.util.EntityUtil.entityReferenceMatch;
import static org.openmetadata.catalog.util.EntityUtil.nextMajorVersion;
import static org.openmetadata.catalog.util.EntityUtil.nextVersion;
import static org.openmetadata.catalog.util.EntityUtil.objectMatch;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
/** /**
* This is the base class used by Entity Resources to perform READ and WRITE operations to the backend database to * This is the base class used by Entity Resources to perform READ and WRITE operations to the backend database to
* Create, Retrieve, Update, and Delete entities. * Create, Retrieve, Update, and Delete entities.
@ -557,34 +558,6 @@ public abstract class EntityRepository<T> {
} }
} }
public EntityReference getOriginalOwner(T entity) throws IOException {
if (!supportsOwner) {
return null;
}
// Try to find the owner if entity exists
try {
String fqn = getFullyQualifiedName(entity);
entity = getByName(null, fqn, getFields(FIELD_OWNER));
return getEntityInterface(entity).getOwner();
} catch (EntityNotFoundException e) {
// If entity is not found, we can return null for owner and ignore this exception
}
return null;
}
protected EntityReference getOwner(T entity) throws IOException {
EntityInterface<T> entityInterface = getEntityInterface(entity);
return getOwner(entityInterface.getId(), entityType);
}
protected void setOwner(T entity, EntityReference owner) {
if (supportsOwner) {
EntityInterface<T> entityInterface = getEntityInterface(entity);
setOwner(entityInterface.getId(), entityType, owner);
entityInterface.setOwner(owner);
}
}
/** Validate given list of tags and add derived tags to it */ /** Validate given list of tags and add derived tags to it */
public final List<TagLabel> addDerivedTags(List<TagLabel> tagLabels) throws IOException { public final List<TagLabel> addDerivedTags(List<TagLabel> tagLabels) throws IOException {
if (tagLabels == null || tagLabels.isEmpty()) { if (tagLabels == null || tagLabels.isEmpty()) {
@ -676,11 +649,16 @@ public abstract class EntityRepository<T> {
} }
protected List<EntityReference> getFollowers(T entity) throws IOException { protected List<EntityReference> getFollowers(T entity) throws IOException {
if (!supportsFollower || entity == null) {
return null;
}
EntityInterface<T> entityInterface = getEntityInterface(entity); EntityInterface<T> entityInterface = getEntityInterface(entity);
return !supportsFollower || entity == null List<EntityReference> followers = findFrom(entityInterface.getId(), entityType, Relationship.FOLLOWS);
? null for (EntityReference follower : followers) {
: EntityUtil.getFollowers( User user = daoCollection.userDAO().findEntityById(follower.getId(), ALL);
entityInterface, entityType, daoCollection.relationshipDAO(), daoCollection.userDAO()); follower.withName(user.getName()).withDeleted(user.getDeleted());
}
return followers;
} }
public T withHref(UriInfo uriInfo, T entity) { public T withHref(UriInfo uriInfo, T entity) {
@ -732,14 +710,6 @@ public abstract class EntityRepository<T> {
return daoCollection.relationshipDAO().insert(from, to, fromEntity, toEntity, relationship.ordinal()); return daoCollection.relationshipDAO().insert(from, to, fromEntity, toEntity, relationship.ordinal());
} }
public void setOwner(UUID ownedEntityId, String ownedEntityType, EntityReference owner) {
// Add relationship owner --- owns ---> ownedEntity
if (owner != null) {
LOG.info("Adding owner {}:{} for entity {}:{}", owner.getType(), owner.getId(), ownedEntityType, ownedEntityId);
addRelationship(owner.getId(), ownedEntityId, owner.getType(), ownedEntityType, Relationship.OWNS);
}
}
public List<String> findBoth(UUID entity1, String entityType1, Relationship relationship, String entity2) { public List<String> findBoth(UUID entity1, String entityType1, Relationship relationship, String entity2) {
// Find bidirectional relationship // Find bidirectional relationship
List<String> ids = new ArrayList<>(); List<String> ids = new ArrayList<>();
@ -779,15 +749,6 @@ public abstract class EntityRepository<T> {
} }
} }
public EntityReference getOwner(UUID id, String entityType) throws IOException {
if (!supportsOwner) {
return null;
}
List<EntityReference> refs = findFrom(id, entityType, Relationship.OWNS);
ensureSingleRelationship(entityType, id, refs, "owners", false);
return refs.isEmpty() ? null : Entity.getEntityReferenceById(refs.get(0).getType(), refs.get(0).getId(), ALL);
}
public List<String> findTo(UUID fromId, String fromEntityType, Relationship relationship, String toEntityType) { public List<String> findTo(UUID fromId, String fromEntityType, Relationship relationship, String toEntityType) {
return daoCollection return daoCollection
.relationshipDAO() .relationshipDAO()
@ -823,6 +784,77 @@ public abstract class EntityRepository<T> {
} }
} }
public EntityReference getOwner(T entity) throws IOException {
if (!supportsOwner) {
return null;
}
EntityInterface<T> entityInterface = getEntityInterface(entity);
List<EntityReference> refs = findFrom(entityInterface.getId(), entityType, Relationship.OWNS);
ensureSingleRelationship(entityType, entityInterface.getId(), refs, "owners", false);
return refs.isEmpty() ? null : getOwner(refs.get(0));
}
public EntityReference getOwner(EntityReference ref) throws IOException {
return Entity.getEntityReferenceById(ref.getType(), ref.getId(), ALL);
}
public EntityReference getOriginalOwner(T entity) throws IOException {
if (!supportsOwner) {
return null;
}
// Try to find the owner if entity exists
try {
String fqn = getFullyQualifiedName(entity);
entity = getByName(null, fqn, getFields(FIELD_OWNER));
return getEntityInterface(entity).getOwner();
} catch (EntityNotFoundException e) {
// If entity is not found, we can return null for owner and ignore this exception
}
return null;
}
public EntityReference populateOwner(EntityReference owner) throws IOException {
if (owner == null) {
return null;
}
EntityReference ref = Entity.getEntityReferenceById(owner.getType(), owner.getId(), ALL);
return owner.withName(ref.getName()).withDisplayName(ref.getDisplayName()).withDeleted(ref.getDeleted());
}
protected void setOwner(T entity, EntityReference owner) {
if (supportsOwner) {
EntityInterface<T> entityInterface = getEntityInterface(entity);
addOwnerRelationship(entityInterface.getId(), entityType, owner);
entityInterface.setOwner(owner);
}
}
public void addOwnerRelationship(UUID ownedEntityId, String ownedEntityType, EntityReference owner) {
// Add relationship owner --- owns ---> ownedEntity
if (owner != null) {
LOG.info("Adding owner {}:{} for entity {}:{}", owner.getType(), owner.getId(), ownedEntityType, ownedEntityId);
addRelationship(owner.getId(), ownedEntityId, owner.getType(), ownedEntityType, Relationship.OWNS);
}
}
/** Remove owner relationship for a given entity */
private void removeOwnerRelationship(EntityReference owner, String ownedEntityId, String ownedEntityType) {
if (owner != null && owner.getId() != null) {
LOG.info("Removing owner {}:{} for entity {}", owner.getType(), owner.getId(), ownedEntityId);
daoCollection
.relationshipDAO()
.delete(
owner.getId().toString(), owner.getType(), ownedEntityId, ownedEntityType, Relationship.OWNS.ordinal());
}
}
public void updateOwnerRelationship(
EntityReference originalOwner, EntityReference newOwner, UUID ownedEntityId, String ownedEntityType) {
// TODO inefficient use replace instead of delete and add and check for orig and new owners being the same
removeOwnerRelationship(originalOwner, ownedEntityId.toString(), ownedEntityType);
addOwnerRelationship(ownedEntityId, ownedEntityType, newOwner);
}
public final Fields getFields(String fields) { public final Fields getFields(String fields) {
return new Fields(allowedFields, fields); return new Fields(allowedFields, fields);
} }
@ -938,8 +970,7 @@ public abstract class EntityRepository<T> {
if (operation.isPatch() || updatedOwner != null) { if (operation.isPatch() || updatedOwner != null) {
// Update owner for all PATCH operations. For PUT operations, ownership can't be removed // Update owner for all PATCH operations. For PUT operations, ownership can't be removed
if (recordChange(FIELD_OWNER, origOwner, updatedOwner, true, entityReferenceMatch)) { if (recordChange(FIELD_OWNER, origOwner, updatedOwner, true, entityReferenceMatch)) {
EntityUtil.updateOwner( updateOwnerRelationship(origOwner, updatedOwner, original.getId(), entityType);
daoCollection.relationshipDAO(), origOwner, updatedOwner, original.getId(), entityType);
} }
} }
} }

View File

@ -35,7 +35,6 @@ import org.openmetadata.catalog.type.TagLabel.Source;
import org.openmetadata.catalog.util.EntityInterface; import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.JsonUtils;
public class GlossaryRepository extends EntityRepository<Glossary> { public class GlossaryRepository extends EntityRepository<Glossary> {
private static final String UPDATE_FIELDS = "owner,tags,reviewers"; private static final String UPDATE_FIELDS = "owner,tags,reviewers";
@ -77,11 +76,7 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
// Don't store owner, href and tags as JSON. Build it on the fly based on relationships // Don't store owner, href and tags as JSON. Build it on the fly based on relationships
glossary.withOwner(null).withHref(null).withTags(null); glossary.withOwner(null).withHref(null).withTags(null);
if (update) { store(glossary.getId(), glossary, update);
daoCollection.glossaryDAO().update(glossary.getId(), JsonUtils.pojoToJson(glossary));
} else {
daoCollection.glossaryDAO().insert(glossary);
}
// Restore the relationships // Restore the relationships
glossary.withOwner(owner).withTags(tags).withReviewers(reviewers); glossary.withOwner(owner).withTags(tags).withReviewers(reviewers);

View File

@ -41,7 +41,6 @@ import org.openmetadata.catalog.type.TagLabel.Source;
import org.openmetadata.catalog.util.EntityInterface; import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.JsonUtils;
@Slf4j @Slf4j
public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> { public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
@ -139,11 +138,7 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
.withHref(null) .withHref(null)
.withTags(null); .withTags(null);
if (update) { store(entity.getId(), entity, update);
daoCollection.glossaryTermDAO().update(entity.getId(), JsonUtils.pojoToJson(entity));
} else {
daoCollection.glossaryTermDAO().insert(entity);
}
// Restore the relationships // Restore the relationships
entity entity

View File

@ -157,11 +157,6 @@ public class LocationRepository extends EntityRepository<Location> {
: null; : null;
} }
@Transaction
public EntityReference getOwnerReference(Location location) throws IOException {
return EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), location.getOwner());
}
private StorageService getService(UUID serviceId, String entityType) throws IOException { private StorageService getService(UUID serviceId, String entityType) throws IOException {
if (entityType.equalsIgnoreCase(Entity.STORAGE_SERVICE)) { if (entityType.equalsIgnoreCase(Entity.STORAGE_SERVICE)) {
return daoCollection.storageServiceDAO().findEntityById(serviceId); return daoCollection.storageServiceDAO().findEntityById(serviceId);
@ -177,7 +172,7 @@ public class LocationRepository extends EntityRepository<Location> {
new StorageServiceRepository.StorageServiceEntityInterface(storageService).getEntityReference()); new StorageServiceRepository.StorageServiceEntityInterface(storageService).getEntityReference());
location.setServiceType(storageService.getServiceType()); location.setServiceType(storageService.getServiceType());
location.setFullyQualifiedName(getFQN(location)); location.setFullyQualifiedName(getFQN(location));
EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), location.getOwner()); // Validate owner populateOwner(location.getOwner()); // Validate owner
location.setTags(addDerivedTags(location.getTags())); location.setTags(addDerivedTags(location.getTags()));
} }
@ -200,7 +195,7 @@ public class LocationRepository extends EntityRepository<Location> {
@Override @Override
public void storeRelationships(Location location) { public void storeRelationships(Location location) {
// Add location owner relationship // Add location owner relationship
setOwner(location.getId(), Entity.LOCATION, location.getOwner()); addOwnerRelationship(location.getId(), Entity.LOCATION, location.getOwner());
EntityReference service = location.getService(); EntityReference service = location.getService();
addRelationship(service.getId(), location.getId(), service.getType(), Entity.LOCATION, Relationship.CONTAINS); addRelationship(service.getId(), location.getId(), service.getType(), Entity.LOCATION, Relationship.CONTAINS);

View File

@ -69,7 +69,7 @@ public class MetricsRepository extends EntityRepository<Metrics> {
@Override @Override
public void prepare(Metrics metrics) throws IOException { public void prepare(Metrics metrics) throws IOException {
metrics.setFullyQualifiedName(getFQN(metrics)); metrics.setFullyQualifiedName(getFQN(metrics));
EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), metrics.getOwner()); // Validate owner populateOwner(metrics.getOwner()); // Validate owner
metrics.setService(getService(metrics.getService())); metrics.setService(getService(metrics.getService()));
metrics.setTags(addDerivedTags(metrics.getTags())); metrics.setTags(addDerivedTags(metrics.getTags()));
} }

View File

@ -27,7 +27,6 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.MlModel; import org.openmetadata.catalog.entity.data.MlModel;
import org.openmetadata.catalog.resources.mlmodels.MlModelResource; import org.openmetadata.catalog.resources.mlmodels.MlModelResource;
@ -62,11 +61,6 @@ public class MlModelRepository extends EntityRepository<MlModel> {
return model.getName(); return model.getName();
} }
@Transaction
public EntityReference getOwnerReference(MlModel mlModel) throws IOException {
return EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), mlModel.getOwner());
}
@Override @Override
public MlModel setFields(MlModel mlModel, Fields fields) throws IOException { public MlModel setFields(MlModel mlModel, Fields fields) throws IOException {
mlModel.setOwner(fields.contains(FIELD_OWNER) ? getOwner(mlModel) : null); mlModel.setOwner(fields.contains(FIELD_OWNER) ? getOwner(mlModel) : null);
@ -141,7 +135,7 @@ public class MlModelRepository extends EntityRepository<MlModel> {
} }
// Check if owner is valid and set the relationship // Check if owner is valid and set the relationship
mlModel.setOwner(EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), mlModel.getOwner())); populateOwner(mlModel.getOwner());
// Check that the dashboard exists // Check that the dashboard exists
if (mlModel.getDashboard() != null) { if (mlModel.getDashboard() != null) {
@ -169,7 +163,7 @@ public class MlModelRepository extends EntityRepository<MlModel> {
@Override @Override
public void storeRelationships(MlModel mlModel) { public void storeRelationships(MlModel mlModel) {
setOwner(mlModel.getId(), Entity.MLMODEL, mlModel.getOwner()); addOwnerRelationship(mlModel.getId(), Entity.MLMODEL, mlModel.getOwner());
setDashboard(mlModel, mlModel.getDashboard()); setDashboard(mlModel, mlModel.getDashboard());

View File

@ -67,11 +67,6 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
: null; : null;
} }
@Transaction
public EntityReference getOwnerReference(Pipeline pipeline) throws IOException {
return EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), pipeline.getOwner());
}
@Override @Override
public Pipeline setFields(Pipeline pipeline, Fields fields) throws IOException { public Pipeline setFields(Pipeline pipeline, Fields fields) throws IOException {
pipeline.setDisplayName(pipeline.getDisplayName()); pipeline.setDisplayName(pipeline.getDisplayName());
@ -160,7 +155,7 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
public void prepare(Pipeline pipeline) throws IOException { public void prepare(Pipeline pipeline) throws IOException {
populateService(pipeline); populateService(pipeline);
pipeline.setFullyQualifiedName(getFQN(pipeline)); pipeline.setFullyQualifiedName(getFQN(pipeline));
EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), pipeline.getOwner()); // Validate owner populateOwner(pipeline.getOwner()); // Validate owner
pipeline.setTags(addDerivedTags(pipeline.getTags())); pipeline.setTags(addDerivedTags(pipeline.getTags()));
} }
@ -186,7 +181,7 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
addRelationship(service.getId(), pipeline.getId(), service.getType(), Entity.PIPELINE, Relationship.CONTAINS); addRelationship(service.getId(), pipeline.getId(), service.getType(), Entity.PIPELINE, Relationship.CONTAINS);
// Add owner relationship // Add owner relationship
setOwner(pipeline.getId(), Entity.PIPELINE, pipeline.getOwner()); addOwnerRelationship(pipeline.getId(), Entity.PIPELINE, pipeline.getOwner());
// Add tag to pipeline relationship // Add tag to pipeline relationship
applyTags(pipeline); applyTags(pipeline);

View File

@ -66,11 +66,6 @@ public class PolicyRepository extends EntityRepository<Policy> {
return policy.getName(); return policy.getName();
} }
@Transaction
public EntityReference getOwnerReference(Policy policy) throws IOException {
return EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), policy.getOwner());
}
/** Find the location to which this policy applies to. * */ /** Find the location to which this policy applies to. * */
@Transaction @Transaction
private EntityReference getLocationForPolicy(Policy policy) throws IOException { private EntityReference getLocationForPolicy(Policy policy) throws IOException {
@ -113,7 +108,7 @@ public class PolicyRepository extends EntityRepository<Policy> {
policy.setFullyQualifiedName(getFQN(policy)); policy.setFullyQualifiedName(getFQN(policy));
policy.setLocation(getLocationReference(policy)); policy.setLocation(getLocationReference(policy));
// Check if owner is valid and set the relationship // Check if owner is valid and set the relationship
policy.setOwner(EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), policy.getOwner())); populateOwner(policy.getOwner());
} }
@Override @Override

View File

@ -588,7 +588,7 @@ public class TableRepository extends EntityRepository<Table> {
addRelationship(table.getDatabase().getId(), table.getId(), DATABASE, TABLE, Relationship.CONTAINS); addRelationship(table.getDatabase().getId(), table.getId(), DATABASE, TABLE, Relationship.CONTAINS);
// Add table owner relationship // Add table owner relationship
setOwner(table.getId(), TABLE, table.getOwner()); addOwnerRelationship(table.getId(), TABLE, table.getOwner());
// Add tag to table relationship // Add tag to table relationship
applyTags(table); applyTags(table);

View File

@ -78,7 +78,7 @@ public class TeamRepository extends EntityRepository<Team> {
@Override @Override
public void prepare(Team team) throws IOException { public void prepare(Team team) throws IOException {
// Check if owner is valid and set the relationship // Check if owner is valid and set the relationship
EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), team.getOwner()); // Validate owner populateOwner(team.getOwner()); // Validate owner
validateUsers(team.getUsers()); validateUsers(team.getUsers());
validateRoles(team.getDefaultRoles()); validateRoles(team.getDefaultRoles());
} }
@ -102,7 +102,7 @@ public class TeamRepository extends EntityRepository<Team> {
@Override @Override
public void storeRelationships(Team team) { public void storeRelationships(Team team) {
// Add team owner relationship // Add team owner relationship
setOwner(team.getId(), TEAM, team.getOwner()); addOwnerRelationship(team.getId(), TEAM, team.getOwner());
for (EntityReference user : listOrEmpty(team.getUsers())) { for (EntityReference user : listOrEmpty(team.getUsers())) {
addRelationship(team.getId(), user.getId(), TEAM, Entity.USER, Relationship.HAS); addRelationship(team.getId(), user.getId(), TEAM, Entity.USER, Relationship.HAS);
} }

View File

@ -13,15 +13,7 @@
package org.openmetadata.catalog.jdbi3; package org.openmetadata.catalog.jdbi3;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Topic; import org.openmetadata.catalog.entity.data.Topic;
import org.openmetadata.catalog.entity.services.MessagingService; import org.openmetadata.catalog.entity.services.MessagingService;
@ -37,6 +29,14 @@ import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.EntityUtil.Fields;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
public class TopicRepository extends EntityRepository<Topic> { public class TopicRepository extends EntityRepository<Topic> {
private static final String TOPIC_UPDATE_FIELDS = "owner,tags"; private static final String TOPIC_UPDATE_FIELDS = "owner,tags";
private static final String TOPIC_PATCH_FIELDS = "owner,tags"; private static final String TOPIC_PATCH_FIELDS = "owner,tags";
@ -58,11 +58,6 @@ public class TopicRepository extends EntityRepository<Topic> {
TOPIC_UPDATE_FIELDS); TOPIC_UPDATE_FIELDS);
} }
@Transaction
public EntityReference getOwnerReference(Topic topic) throws IOException {
return EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), topic.getOwner());
}
@Override @Override
public void prepare(Topic topic) throws IOException { public void prepare(Topic topic) throws IOException {
MessagingService messagingService = Entity.getEntity(topic.getService(), Fields.EMPTY_FIELDS, Include.ALL); MessagingService messagingService = Entity.getEntity(topic.getService(), Fields.EMPTY_FIELDS, Include.ALL);

View File

@ -13,8 +13,37 @@
package org.openmetadata.catalog.util; package org.openmetadata.catalog.util;
import static org.openmetadata.catalog.type.Include.ALL; import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.joda.time.Period;
import org.joda.time.format.ISOPeriodFormat;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.TermReference;
import org.openmetadata.catalog.entity.data.GlossaryTerm;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityVersionPair;
import org.openmetadata.catalog.jdbi3.CollectionDAO.UsageDAO;
import org.openmetadata.catalog.resources.feeds.MessageParser.EntityLink;
import org.openmetadata.catalog.type.ChangeEvent;
import org.openmetadata.catalog.type.Column;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.EventFilter;
import org.openmetadata.catalog.type.EventType;
import org.openmetadata.catalog.type.FailureDetails;
import org.openmetadata.catalog.type.FieldChange;
import org.openmetadata.catalog.type.MlFeature;
import org.openmetadata.catalog.type.MlHyperParameter;
import org.openmetadata.catalog.type.Schedule;
import org.openmetadata.catalog.type.TableConstraint;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.Task;
import org.openmetadata.catalog.type.UsageDetails;
import org.openmetadata.catalog.type.UsageStats;
import javax.ws.rs.WebApplicationException;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -27,42 +56,8 @@ import java.util.UUID;
import java.util.function.BiPredicate; import java.util.function.BiPredicate;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import javax.ws.rs.WebApplicationException;
import lombok.NonNull; import static org.openmetadata.catalog.type.Include.ALL;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.joda.time.Period;
import org.joda.time.format.ISOPeriodFormat;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.TermReference;
import org.openmetadata.catalog.entity.data.GlossaryTerm;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.entity.teams.Team;
import org.openmetadata.catalog.entity.teams.User;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityRelationshipDAO;
import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityVersionPair;
import org.openmetadata.catalog.jdbi3.CollectionDAO.TeamDAO;
import org.openmetadata.catalog.jdbi3.CollectionDAO.UsageDAO;
import org.openmetadata.catalog.jdbi3.CollectionDAO.UserDAO;
import org.openmetadata.catalog.resources.feeds.MessageParser.EntityLink;
import org.openmetadata.catalog.type.ChangeEvent;
import org.openmetadata.catalog.type.Column;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.EventFilter;
import org.openmetadata.catalog.type.EventType;
import org.openmetadata.catalog.type.FailureDetails;
import org.openmetadata.catalog.type.FieldChange;
import org.openmetadata.catalog.type.MlFeature;
import org.openmetadata.catalog.type.MlHyperParameter;
import org.openmetadata.catalog.type.Relationship;
import org.openmetadata.catalog.type.Schedule;
import org.openmetadata.catalog.type.TableConstraint;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.Task;
import org.openmetadata.catalog.type.UsageDetails;
import org.openmetadata.catalog.type.UsageStats;
@Slf4j @Slf4j
public final class EntityUtil { public final class EntityUtil {
@ -167,66 +162,6 @@ public final class EntityUtil {
return entity; return entity;
} }
public static void validateUser(UserDAO userDAO, UUID userId) {
if (!userDAO.exists(userId)) {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Entity.USER, userId));
}
}
public static EntityReference populateOwner(UserDAO userDAO, TeamDAO teamDAO, EntityReference owner)
throws IOException {
if (owner == null) {
return null;
}
UUID id = owner.getId();
if (owner.getType().equalsIgnoreCase("user")) {
User ownerInstance = userDAO.findEntityById(id);
owner.setName(ownerInstance.getName());
owner.setDisplayName(ownerInstance.getDisplayName());
if (Optional.ofNullable(ownerInstance.getDeleted()).orElse(false)) {
throw new IllegalArgumentException(CatalogExceptionMessage.deactivatedUser(id));
}
} else if (owner.getType().equalsIgnoreCase("team")) {
Team ownerInstance = teamDAO.findEntityById(id);
owner.setDescription(ownerInstance.getDescription());
owner.setName(ownerInstance.getName());
owner.setDisplayName(ownerInstance.getDisplayName());
} else {
throw new IllegalArgumentException(String.format("Invalid ownerType %s", owner.getType()));
}
return owner;
}
public static void setOwner(
EntityRelationshipDAO dao, UUID ownedEntityId, String ownedEntityType, EntityReference owner) {
// Add relationship owner --- owns ---> ownedEntity
if (owner != null) {
LOG.info("Adding owner {}:{} for entity {}:{}", owner.getType(), owner.getId(), ownedEntityType, ownedEntityId);
dao.insert(owner.getId(), ownedEntityId, owner.getType(), ownedEntityType, Relationship.OWNS.ordinal());
}
}
/** Unassign owner relationship for a given entity */
public static void unassignOwner(
EntityRelationshipDAO dao, EntityReference owner, String ownedEntityId, String ownedEntityType) {
if (owner != null && owner.getId() != null) {
LOG.info("Removing owner {}:{} for entity {}", owner.getType(), owner.getId(), ownedEntityId);
dao.delete(
owner.getId().toString(), owner.getType(), ownedEntityId, ownedEntityType, Relationship.OWNS.ordinal());
}
}
public static void updateOwner(
EntityRelationshipDAO dao,
EntityReference originalOwner,
EntityReference newOwner,
UUID ownedEntityId,
String ownedEntityType) {
// TODO inefficient use replace instead of delete and add and check for orig and new owners being the same
unassignOwner(dao, originalOwner, ownedEntityId.toString(), ownedEntityType);
setOwner(dao, ownedEntityId, ownedEntityType, newOwner);
}
public static List<EntityReference> populateEntityReferences(List<EntityReference> list) throws IOException { public static List<EntityReference> populateEntityReferences(List<EntityReference> list) throws IOException {
if (list != null) { if (list != null) {
for (EntityReference ref : list) { for (EntityReference ref : list) {
@ -284,41 +219,6 @@ public final class EntityUtil {
} }
} }
public static boolean addFollower(
EntityRelationshipDAO dao,
UserDAO userDAO,
UUID followedEntityId,
String followedEntityType,
UUID followerId,
String followerEntity)
throws IOException {
User user = userDAO.findEntityById(followerId);
if (Optional.ofNullable(user.getDeleted()).orElse(false)) {
throw new IllegalArgumentException(CatalogExceptionMessage.deactivatedUser(followerId));
}
return dao.insert(followerId, followedEntityId, followerEntity, followedEntityType, Relationship.FOLLOWS.ordinal())
> 0;
}
public static List<EntityReference> getFollowers(
EntityInterface<?> followedEntityInterface,
String name,
EntityRelationshipDAO entityRelationshipDAO,
UserDAO userDAO)
throws IOException {
List<String> followerIds =
entityRelationshipDAO.findFrom(
followedEntityInterface.getId().toString(), name, Relationship.FOLLOWS.ordinal(), Entity.USER);
List<EntityReference> followers = new ArrayList<>();
for (String followerId : followerIds) {
User user = userDAO.findEntityById(UUID.fromString(followerId), ALL);
if (followedEntityInterface.isDeleted() || !user.getDeleted()) {
followers.add(new EntityReference().withName(user.getName()).withId(user.getId()).withType("user"));
}
}
return followers;
}
@RequiredArgsConstructor @RequiredArgsConstructor
public static class Fields { public static class Fields {
public static final Fields EMPTY_FIELDS = new Fields(null, null); public static final Fields EMPTY_FIELDS = new Fields(null, null);
@ -396,7 +296,7 @@ public final class EntityUtil {
return String.join(Entity.SEPARATOR, strings); return String.join(Entity.SEPARATOR, strings);
} }
/** Return column field name of format columnName.fieldName */ /** Return column field name of format columnName:fieldName */
public static String getColumnField(Table table, Column column, String columnField) { public static String getColumnField(Table table, Column column, String columnField) {
// Remove table FQN from column FQN to get the local name // Remove table FQN from column FQN to get the local name
String localColumnName = String localColumnName =

View File

@ -16,6 +16,7 @@ package org.openmetadata.catalog.resources.events;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS; import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
import java.io.IOException; import java.io.IOException;
@ -329,6 +330,7 @@ public class WebhookResourceTest extends EntityResourceTest<Webhook, CreateWebho
// For the entity all the webhooks registered for created events have the right number of events // For the entity all the webhooks registered for created events have the right number of events
List<ChangeEvent> callbackEvents = List<ChangeEvent> callbackEvents =
webhookCallbackResource.getEntityCallbackEvents(EventType.ENTITY_CREATED, entity); webhookCallbackResource.getEntityCallbackEvents(EventType.ENTITY_CREATED, entity);
assertTrue(callbackEvents.size() > 1);
long timestamp = callbackEvents.get(0).getTimestamp(); long timestamp = callbackEvents.get(0).getTimestamp();
waitAndCheckForEvents(entity, null, null, timestamp, callbackEvents, 30, 100); waitAndCheckForEvents(entity, null, null, timestamp, callbackEvents, 30, 100);

View File

@ -310,16 +310,21 @@ public final class TestUtils {
} }
public static void existsInEntityReferenceList(List<EntityReference> list, UUID id, boolean expectedExistsInList) { public static void existsInEntityReferenceList(List<EntityReference> list, UUID id, boolean expectedExistsInList) {
boolean exists = false; EntityReference ref = null;
for (EntityReference ref : list) { for (EntityReference r : list) {
validateEntityReference(ref); validateEntityReference(r);
if (ref.getId().equals(id)) { if (r.getId().equals(id)) {
exists = true; ref = r;
break; break;
} }
} }
assertEquals( if (expectedExistsInList) {
expectedExistsInList, exists, "Entry exists in list - expected:" + expectedExistsInList + " actual:" + exists); assertNotNull(ref, "EntityReference does not exist for " + id);
} else {
if (ref != null) {
assertTrue(ref.getDeleted(), "EntityReference is not deleted as expected " + id);
}
}
} }
public static void assertListNull(Object... values) { public static void assertListNull(Object... values) {