yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-26 15:10:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #1961 - Delete API soft deletes the entities (#1968)

Browse Source
This commit is contained in:
Suresh Srinivas 2021-12-29 14:20:31 -08:00 committed by GitHub
parent 4a4124d45d
commit 6d7a496aa4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
70 changed files with 446 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
bootstrap/sql/mysql/v003__create_db_connection_info.sql
View File

@ -1,8 +1,116 @@
CREATE TABLE IF NOT EXISTS webhook_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
json JSON NOT NULL,
PRIMARY KEY (id),
UNIQUE KEY unique_name(name)
-- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
);
);
--
-- Change timestamp column precision to include microseconds
--
ALTER TABLE change_event
DROP INDEX dateTime,
DROP COLUMN dateTime,
ADD COLUMN dateTime TIMESTAMP(6) GENERATED ALWAYS AS (STR_TO_DATE(json ->> '$.dateTime', '%Y-%m-%dT%T.%fZ')) NOT NULL
AFTER username,
ADD INDEX (dateTime);
--
-- Update to add deleted fields to data entities
--
ALTER TABLE dbservice_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE messaging_service_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE dashboard_service_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE pipeline_service_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE storage_service_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE database_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE table_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE metric_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE report_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE dashboard_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE ml_model_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE pipeline_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE topic_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE chart_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE location_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE bot_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE policy_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE ingestion_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE team_entity
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);
ALTER TABLE entity_relationship
ADD COLUMN deleted BOOLEAN NOT NULL DEFAULT 0,
ADD INDEX (deleted);
-- Change "team -- contains --> user" relationship to "team -- has --> user" relationship
UPDATE entity_relationship
SET relation = 10 WHERE fromEntity = 'team' AND toEntity = 'user' AND relation = 0;
-- Remove user.deactivated field and use deleted instead
UPDATE user_entity
SET json = JSON_REMOVE(user_entity.json, '$.deactivated');
ALTER TABLE user_entity
DROP COLUMN deactivated,
ADD COLUMN deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
ADD INDEX (deleted);

8
bootstrap/sql/mysql/v004__create_db_connection_info.sql
View File

@ -1,8 +0,0 @@
--
-- Change timestamp column precision to include microseconds
--
ALTER TABLE change_event
DROP INDEX dateTime,
DROP COLUMN dateTime,
ADD COLUMN dateTime TIMESTAMP(6) GENERATED ALWAYS AS (STR_TO_DATE(json ->> '$.dateTime', '%Y-%m-%dT%T.%fZ')) NOT NULL AFTER username,
ADD INDEX (dateTime);

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java
View File

@ -185,6 +185,11 @@ public class BotsRepository extends EntityRepository<Bots> {
@Override
public void setOwner(EntityReference owner) {}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Bots withHref(URI href) {
return entity.withHref(href);

15
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java
View File

@ -19,7 +19,6 @@ import java.net.URI;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Chart;
import org.openmetadata.catalog.entity.services.DashboardService;
@ -55,15 +54,6 @@ public class ChartRepository extends EntityRepository<Chart> {
return (chart.getService().getName() + "." + chart.getName());
}
@Transaction
public void delete(UUID id) {
if (dao.relationshipDAO().findToCount(id.toString(), Relationship.CONTAINS.ordinal(), Entity.CHART) > 0) {
throw new IllegalArgumentException("Chart is not empty");
}
dao.chartDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Override
public void prepare(Chart chart) throws IOException {
populateService(chart);
@ -289,6 +279,11 @@ public class ChartRepository extends EntityRepository<Chart> {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Chart withHref(URI href) {
return entity.withHref(href);

28
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java
View File

@ -336,21 +336,22 @@ public interface CollectionDAO {
//
@SqlQuery(
"SELECT toId, toEntity FROM entity_relationship "
+ "WHERE fromId = :fromId AND relation = :relation "
+ "WHERE fromId = :fromId AND relation = :relation AND deleted = false "
+ "ORDER BY toId")
@RegisterRowMapper(ToEntityReferenceMapper.class)
List<EntityReference> findTo(@Bind("fromId") String fromId, @Bind("relation") int relation);
@SqlQuery(
"SELECT toId FROM entity_relationship "
+ "WHERE fromId = :fromId AND relation = :relation AND toEntity = :toEntity "
+ "WHERE fromId = :fromId AND relation = :relation AND toEntity = :toEntity AND deleted = false "
+ "ORDER BY toId")
List<String> findTo(
@Bind("fromId") String fromId, @Bind("relation") int relation, @Bind("toEntity") String toEntity);
@SqlQuery(
"SELECT count(*) FROM entity_relationship "
+ "WHERE fromId = :fromId AND relation = :relation AND toEntity = :toEntity "
+ "WHERE fromId = :fromId AND relation = :relation AND (toEntity = :toEntity || :toEntity IS NULL) "
+ "AND deleted = false "
+ "ORDER BY fromId")
int findToCount(@Bind("fromId") String fromId, @Bind("relation") int relation, @Bind("toEntity") String toEntity);
@ -359,21 +360,21 @@ public interface CollectionDAO {
//
@SqlQuery(
"SELECT fromId FROM entity_relationship "
+ "WHERE toId = :toId AND relation = :relation AND fromEntity = :fromEntity "
+ "WHERE toId = :toId AND relation = :relation AND fromEntity = :fromEntity AND deleted = false "
+ "ORDER BY fromId")
List<String> findFrom(
@Bind("toId") String toId, @Bind("relation") int relation, @Bind("fromEntity") String fromEntity);
@SqlQuery(
"SELECT fromId, fromEntity FROM entity_relationship "
+ "WHERE toId = :toId AND relation = :relation "
+ "WHERE toId = :toId AND relation = :relation AND deleted = false "
+ "ORDER BY fromId")
@RegisterRowMapper(FromEntityReferenceMapper.class)
List<EntityReference> findFrom(@Bind("toId") String toId, @Bind("relation") int relation);
@SqlQuery(
"SELECT fromId, fromEntity FROM entity_relationship "
+ "WHERE toId = :toId AND relation = :relation AND fromEntity = :fromEntity "
+ "WHERE toId = :toId AND relation = :relation AND fromEntity = :fromEntity AND deleted = false "
+ "ORDER BY fromId")
@RegisterRowMapper(FromEntityReferenceMapper.class)
List<EntityReference> findFromEntity(
@ -401,6 +402,9 @@ public interface CollectionDAO {
@SqlUpdate("DELETE from entity_relationship " + "WHERE toId = :id OR fromId = :id")
void deleteAll(@Bind("id") String id);
@SqlUpdate("UPDATE entity_relationship SET deleted = true WHERE toId = :id OR fromId = :id")
void softDeleteAll(@Bind("id") String id);
}
interface FeedDAO {
@ -1042,17 +1046,17 @@ public interface CollectionDAO {
String findByEmail(@Bind("email") String email);
default int listCount(String team) {
return listCount(getTableName(), getNameColumn(), team, Relationship.CONTAINS.ordinal());
return listCount(getTableName(), getNameColumn(), team, Relationship.HAS.ordinal());
}
@Override
default List<String> listBefore(String team, int limit, String before) {
return listBefore(getTableName(), getNameColumn(), team, limit, before, Relationship.CONTAINS.ordinal());
return listBefore(getTableName(), getNameColumn(), team, limit, before, Relationship.HAS.ordinal());
}
@Override
default List<String> listAfter(String team, int limit, String after) {
return listAfter(getTableName(), getNameColumn(), team, limit, after, Relationship.CONTAINS.ordinal());
return listAfter(getTableName(), getNameColumn(), team, limit, after, Relationship.HAS.ordinal());
}
@SqlQuery(
@ -1061,7 +1065,7 @@ public interface CollectionDAO {
+ "FROM user_entity ue "
+ "LEFT JOIN entity_relationship er on ue.id = er.toId "
+ "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation "
+ "WHERE (te.name = :team OR :team IS NULL) "
+ "WHERE (ue.deleted = false AND (te.name = :team OR :team IS NULL)) "
+ "GROUP BY ue.id) subquery")
int listCount(
@Define("table") String table,
@ -1075,7 +1079,7 @@ public interface CollectionDAO {
+ "FROM user_entity ue "
+ "LEFT JOIN entity_relationship er on ue.id = er.toId "
+ "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation "
+ "WHERE (te.name = :team OR :team IS NULL) AND "
+ "WHERE (ue.deleted = false AND (te.name = :team OR :team IS NULL)) AND "
+ "ue.<nameColumn> < :before "
+ "GROUP BY ue.<nameColumn>, ue.json "
+ "ORDER BY ue.<nameColumn> DESC "
@ -1094,7 +1098,7 @@ public interface CollectionDAO {
+ "FROM user_entity ue "
+ "LEFT JOIN entity_relationship er on ue.id = er.toId "
+ "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation "
+ "WHERE (te.name = :team OR :team IS NULL) AND "
+ "WHERE (ue.deleted = false AND (te.name = :team OR :team IS NULL)) AND "
+ "ue.<nameColumn> > :after "
+ "GROUP BY ue.json "
+ "ORDER BY ue.<nameColumn> "

14
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java
View File

@ -63,15 +63,6 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
return new DashboardEntityInterface(entity);
}
@Transaction
public void delete(UUID id) {
if (dao.relationshipDAO().findToCount(id.toString(), Relationship.CONTAINS.ordinal(), Entity.DASHBOARD) > 0) {
throw new IllegalArgumentException("Dashboard is not empty");
}
dao.dashboardDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Transaction
public EntityReference getOwnerReference(Dashboard dashboard) throws IOException {
return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), dashboard.getOwner());
@ -377,6 +368,11 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Dashboard withHref(URI href) {
return entity.withHref(href);

12
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java
View File

@ -20,7 +20,6 @@ import java.util.Date;
import java.util.List;
import java.util.UUID;
import javax.ws.rs.core.UriInfo;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.services.DashboardService;
import org.openmetadata.catalog.resources.services.dashboard.DashboardServiceResource;
@ -70,12 +69,6 @@ public class DashboardServiceRepository extends EntityRepository<DashboardServic
return withHref(uriInfo, dashboardService);
}
@Transaction
public void delete(UUID id) {
dao.dashboardServiceDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Override
public DashboardService setFields(DashboardService entity, Fields fields) {
return entity;
@ -223,6 +216,11 @@ public class DashboardServiceRepository extends EntityRepository<DashboardServic
@Override
public void setOwner(EntityReference owner) {}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public DashboardService withHref(URI href) {
return entity.withHref(href);

14
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseRepository.java
View File

@ -58,15 +58,6 @@ public class DatabaseRepository extends EntityRepository<Database> {
return (database.getService().getName() + "." + database.getName());
}
@Transaction
public void delete(UUID id) {
if (dao.relationshipDAO().findToCount(id.toString(), Relationship.CONTAINS.ordinal(), Entity.TABLE) > 0) {
throw new IllegalArgumentException("Database is not empty");
}
dao.databaseDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Transaction
public void deleteLocation(String databaseId) {
dao.relationshipDAO().deleteFrom(databaseId, Relationship.HAS.ordinal(), Entity.LOCATION);
@ -312,6 +303,11 @@ public class DatabaseRepository extends EntityRepository<Database> {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Database withHref(URI href) {
return entity.withHref(href);

12
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java
View File

@ -19,7 +19,6 @@ import java.net.URI;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.services.DatabaseService;
import org.openmetadata.catalog.resources.services.database.DatabaseServiceResource;
@ -48,12 +47,6 @@ public class DatabaseServiceRepository extends EntityRepository<DatabaseService>
this.dao = dao;
}
@Transaction
public void delete(UUID id) {
dao.dbServiceDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Override
public DatabaseService setFields(DatabaseService entity, Fields fields) {
return entity;
@ -201,6 +194,11 @@ public class DatabaseServiceRepository extends EntityRepository<DatabaseService>
@Override
public void setOwner(EntityReference owner) {}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public DatabaseService withHref(URI href) {
return entity.withHref(href);

16
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityDAO.java
View File

@ -46,14 +46,15 @@ public interface EntityDAO<T> {
@SqlUpdate("UPDATE <table> SET json = :json WHERE id = :id")
void update(@Define("table") String table, @Bind("id") String id, @Bind("json") String json);
@SqlQuery("SELECT json FROM <table> WHERE id = :id")
@SqlQuery("SELECT json FROM <table> WHERE id = :id AND deleted IS NOT TRUE")
String findById(@Define("table") String table, @Bind("id") String id);
@SqlQuery("SELECT json FROM <table> WHERE <nameColumn> = :name")
@SqlQuery("SELECT json FROM <table> WHERE <nameColumn> = :name AND deleted IS NOT TRUE")
String findByName(@Define("table") String table, @Define("nameColumn") String nameColumn, @Bind("name") String name);
@SqlQuery(
"SELECT count(*) FROM <table> WHERE " + "(<nameColumn> LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)")
"SELECT count(*) FROM <table> WHERE "
+ "(<nameColumn> LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL) AND deleted IS NOT TRUE")
int listCount(
@Define("table") String table, @Define("nameColumn") String nameColumn, @Bind("fqnPrefix") String fqnPrefix);
@ -61,9 +62,9 @@ public interface EntityDAO<T> {
"SELECT json FROM ("
+ "SELECT <nameColumn>, json FROM <table> WHERE "
+ "(<nameColumn> LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL) AND "
+ // Filter by
// service name
"<nameColumn> < :before "
+ // Filter by service name
"<nameColumn> < :before AND "
+ "deleted = false "
+ // Pagination by chart fullyQualifiedName
"ORDER BY <nameColumn> DESC "
+ // Pagination ordering by chart fullyQualifiedName
@ -79,7 +80,8 @@ public interface EntityDAO<T> {
@SqlQuery(
"SELECT json FROM <table> WHERE "
+ "(<nameColumn> LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL) AND "
+ "<nameColumn> > :after "
+ "<nameColumn> > :after AND "
+ "deleted = false "
+ "ORDER BY <nameColumn> "
+ "LIMIT :limit")
List<String> listAfter(

22
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java
View File

@ -97,6 +97,7 @@ public abstract class EntityRepository<T> {
private final String entityName;
protected final EntityDAO<T> dao;
protected final CollectionDAO daoCollection;
protected boolean softDelete = true;
/** Fields that can be updated during PATCH operation */
private final Fields patchFields;
@ -331,7 +332,7 @@ public abstract class EntityRepository<T> {
// Validate follower
User user = daoCollection.userDAO().findEntityById(userId);
if (user.getDeactivated()) {
if (user.getDeleted()) {
throw new IllegalArgumentException(CatalogExceptionMessage.deactivatedUser(userId));
}
@ -360,6 +361,25 @@ public abstract class EntityRepository<T> {
return new PutResponse<>(added > 0 ? Status.CREATED : Status.OK, changeEvent, RestUtil.ENTITY_FIELDS_CHANGED);
}
@Transaction
public final void delete(UUID id) throws IOException {
// If an entity being deleted contains other children entities, it can't be deleted
if (daoCollection.relationshipDAO().findToCount(id.toString(), Relationship.CONTAINS.ordinal(), null) > 0) {
throw new IllegalArgumentException(entityName + " is not empty");
}
if (softDelete) {
T entity = dao.findEntityById(id);
EntityInterface<T> entityInterface = getEntityInterface(entity);
entityInterface.setDeleted(true);
storeEntity(entity, true);
daoCollection.relationshipDAO().softDeleteAll(id.toString());
return;
}
// Hard delete
dao.delete(id);
daoCollection.relationshipDAO().deleteAll(id.toString());
}
@Transaction
public PutResponse<T> deleteFollower(String updatedBy, UUID entityId, UUID userId) throws IOException {
T entity = dao.findEntityById(entityId);

14
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java
View File

@ -55,15 +55,6 @@ public class IngestionRepository extends EntityRepository<Ingestion> {
return (ingestion.getService().getName() + "." + ingestion.getName());
}
@Transaction
public void delete(UUID id) {
if (dao.relationshipDAO().findToCount(id.toString(), Relationship.CONTAINS.ordinal(), Entity.INGESTION) > 0) {
throw new IllegalArgumentException("Ingestion is not empty");
}
dao.ingestionDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Transaction
public EntityReference getOwnerReference(Ingestion ingestion) throws IOException {
return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), ingestion.getOwner());
@ -287,6 +278,11 @@ public class IngestionRepository extends EntityRepository<Ingestion> {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Ingestion withHref(URI href) {
return entity.withHref(href);

11
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java
View File

@ -149,12 +149,6 @@ public class LocationRepository extends EntityRepository<Location> {
return (location.getService().getName() + "." + location.getName());
}
@Transaction
public void delete(UUID id) {
dao.locationDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString()); // Remove all relationships
}
@Transaction
public EntityReference getOwnerReference(Location location) throws IOException {
return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), location.getOwner());
@ -380,6 +374,11 @@ public class LocationRepository extends EntityRepository<Location> {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Location withHref(URI href) {
return entity.withHref(href);

12
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java
View File

@ -20,7 +20,6 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.services.MessagingService;
import org.openmetadata.catalog.resources.services.messaging.MessagingServiceResource;
@ -47,12 +46,6 @@ public class MessagingServiceRepository extends EntityRepository<MessagingServic
this.dao = dao;
}
@Transaction
public void delete(UUID id) {
dao.messagingServiceDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Override
public MessagingService setFields(MessagingService entity, Fields fields) {
return entity;
@ -200,6 +193,11 @@ public class MessagingServiceRepository extends EntityRepository<MessagingServic
@Override
public void setOwner(EntityReference owner) {}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public MessagingService withHref(URI href) {
return entity.withHref(href);

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MetricsRepository.java
View File

@ -254,6 +254,11 @@ public class MetricsRepository extends EntityRepository<Metrics> {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Metrics withHref(URI href) {
return entity.withHref(href);

11
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java
View File

@ -66,12 +66,6 @@ public class MlModelRepository extends EntityRepository<MlModel> {
return (model.getName());
}
@Transaction
public void delete(UUID id) {
dao.mlModelDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Transaction
public EntityReference getOwnerReference(MlModel mlModel) throws IOException {
return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), mlModel.getOwner());
@ -373,6 +367,11 @@ public class MlModelRepository extends EntityRepository<MlModel> {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public MlModel withHref(URI href) {
return entity.withHref(href);

14
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java
View File

@ -61,15 +61,6 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
return (pipeline.getService().getName() + "." + pipeline.getName());
}
@Transaction
public void delete(UUID id) {
if (dao.relationshipDAO().findToCount(id.toString(), Relationship.CONTAINS.ordinal(), Entity.PIPELINE) > 0) {
throw new IllegalArgumentException("Pipeline is not empty");
}
dao.pipelineDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Transaction
public EntityReference getOwnerReference(Pipeline pipeline) throws IOException {
return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), pipeline.getOwner());
@ -312,6 +303,11 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Pipeline withHref(URI href) {
return entity.withHref(href);

12
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java
View File

@ -18,7 +18,6 @@ import java.net.URI;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.services.PipelineService;
import org.openmetadata.catalog.resources.services.pipeline.PipelineServiceResource;
@ -45,12 +44,6 @@ public class PipelineServiceRepository extends EntityRepository<PipelineService>
this.dao = dao;
}
@Transaction
public void delete(UUID id) {
dao.pipelineServiceDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Override
public PipelineService setFields(PipelineService entity, Fields fields) {
return entity;
@ -198,6 +191,11 @@ public class PipelineServiceRepository extends EntityRepository<PipelineService>
@Override
public void setOwner(EntityReference owner) {}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public PipelineService withHref(URI href) {
return entity.withHref(href);

14
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java
View File

@ -56,15 +56,6 @@ public class PolicyRepository extends EntityRepository<Policy> {
return (policy.getName());
}
@Transaction
public void delete(UUID id) {
if (dao.relationshipDAO().findToCount(id.toString(), Relationship.CONTAINS.ordinal(), Entity.POLICY) > 0) {
throw new IllegalArgumentException("Policy is not empty");
}
dao.policyDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Transaction
public EntityReference getOwnerReference(Policy policy) throws IOException {
return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), policy.getOwner());
@ -298,6 +289,11 @@ public class PolicyRepository extends EntityRepository<Policy> {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
public void setRules(List<Object> rules) {
entity.setRules(rules);
}

29
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/Relationship.java
View File

@ -24,10 +24,18 @@ public enum Relationship {
* database might have stored the enum ordinal number - When adding a new enum, add it as the last enum to preserve
* the ordinal positions of the existing enums
*/
// Database --- contains --> Table
// Organization --- contains --> Team
// Team --- contains --> User
// Service --- contains --> Database
/**
* CONTAINS relationship is a stronger relationship than HAS. The entity that contains other entities can't be deleted
* until all the entities that it contains are also deleted. Some examples of these relationships:
*
* <ul>
* <li>Database --- contains --> Table
* <li>Organization --- contains --> Team
* <li>Team --- contains --> User
* <li>Service --- contains --> Database
* </ul>
*/
CONTAINS("contains"), // 0
// User/Bot --- created ---> Thread
@ -59,9 +67,16 @@ public enum Relationship {
// {Role} --- parentOf ---> {Role}
PARENT_OF("parentOf"), // 9
// {User} --- has ---> {Role}
// {Table} --- has ---> {Location}
// {Database} --- has ---> {Location}
/**
* HAS relationship is a weaker relationship compared to CONTAINS relationship. The entity that has HAS another entity
* can be deleted. During deletion, the HAS relationship is simply deleted. Examples of HAS relationship:
*
* <ul>
* <li>{User} --- has ---> {Role}
* <li>{Table} --- has ---> {Location}
* <li>{Database} --- has ---> {Location}
* </ul>
*/
HAS("has"), // 10
// {User} --- follows ----> {Table, Database, Metrics...}

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java
View File

@ -227,6 +227,11 @@ public class ReportRepository extends EntityRepository<Report> {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Report withHref(URI href) {
return entity.withHref(href);

12
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java
View File

@ -20,7 +20,6 @@ import java.net.URI;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.services.StorageService;
import org.openmetadata.catalog.resources.services.storage.StorageServiceResource;
@ -45,12 +44,6 @@ public class StorageServiceRepository extends EntityRepository<StorageService> {
this.dao = dao;
}
@Transaction
public void delete(UUID id) {
dao.storageServiceDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Override
public StorageService setFields(StorageService entity, Fields fields) {
return entity;
@ -191,6 +184,11 @@ public class StorageServiceRepository extends EntityRepository<StorageService> {
@Override
public void setOwner(EntityReference owner) {}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public StorageService withHref(URI href) {
return entity.withHref(href);

11
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java
View File

@ -128,12 +128,6 @@ public class TableRepository extends EntityRepository<Table> {
return (table.getDatabase().getName() + "." + table.getName());
}
@Transaction
public void delete(UUID id) {
dao.tableDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString()); // Remove all relationships
}
@Transaction
public Table addJoins(UUID tableId, TableJoins joins) throws IOException, ParseException {
// Validate the request content
@ -791,6 +785,11 @@ public class TableRepository extends EntityRepository<Table> {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Table withHref(URI href) {
return entity.withHref(href);

29
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java
View File

@ -25,7 +25,6 @@ import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.teams.Team;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
@ -55,16 +54,6 @@ public class TeamRepository extends EntityRepository<Team> {
this.dao = dao;
}
@Transaction
public void delete(UUID id) {
// Query 1 - delete team
dao.teamDAO().delete(id);
// Query 2 - Remove all relationship from and to this team
// TODO make this UUID based
dao.relationshipDAO().deleteAll(id.toString());
}
public List<EntityReference> getUsers(List<UUID> userIds) {
if (userIds == null) {
return null;
@ -133,7 +122,8 @@ public class TeamRepository extends EntityRepository<Team> {
public void storeRelationships(Team team) {
for (EntityReference user : Optional.ofNullable(team.getUsers()).orElse(Collections.emptyList())) {
dao.relationshipDAO()
.insert(team.getId().toString(), user.getId().toString(), "team", "user", Relationship.CONTAINS.ordinal());
.insert(team.getId().toString(), user.getId().toString(), "team", "user", Relationship.HAS.ordinal());
System.out.println("Team " + team.getName() + " has user " + user.getName());
}
}
@ -143,7 +133,7 @@ public class TeamRepository extends EntityRepository<Team> {
}
private List<EntityReference> getUsers(String id) throws IOException {
List<String> userIds = dao.relationshipDAO().findTo(id, Relationship.CONTAINS.ordinal(), "user");
List<String> userIds = dao.relationshipDAO().findTo(id, Relationship.HAS.ordinal(), "user");
List<EntityReference> users = new ArrayList<>();
for (String userId : userIds) {
users.add(dao.userDAO().findEntityReferenceById(UUID.fromString(userId)));
@ -264,6 +254,11 @@ public class TeamRepository extends EntityRepository<Team> {
@Override
public void setOwner(EntityReference owner) {}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Team withHref(URI href) {
return entity.withHref(href);
@ -302,16 +297,12 @@ public class TeamRepository extends EntityRepository<Team> {
List<EntityReference> deleted = new ArrayList<>();
if (recordListChange("users", origUsers, updatedUsers, added, deleted, entityReferenceMatch)) {
// Remove users from original and add users from updated
dao.relationshipDAO().deleteFrom(origTeam.getId().toString(), Relationship.CONTAINS.ordinal(), "user");
dao.relationshipDAO().deleteFrom(origTeam.getId().toString(), Relationship.HAS.ordinal(), "user");
// Add relationships
for (EntityReference user : updatedUsers) {
dao.relationshipDAO()
.insert(
updatedTeam.getId().toString(),
user.getId().toString(),
"team",
"user",
Relationship.CONTAINS.ordinal());
updatedTeam.getId().toString(), user.getId().toString(), "team", "user", Relationship.HAS.ordinal());
}
updatedUsers.sort(EntityUtil.compareEntityReference);

14
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java
View File

@ -54,15 +54,6 @@ public class TopicRepository extends EntityRepository<Topic> {
this.dao = dao;
}
@Transaction
public void delete(UUID id) {
if (dao.relationshipDAO().findToCount(id.toString(), Relationship.CONTAINS.ordinal(), Entity.TOPIC) > 0) {
throw new IllegalArgumentException("Topic is not empty");
}
dao.topicDAO().delete(id);
dao.relationshipDAO().deleteAll(id.toString());
}
@Transaction
public EntityReference getOwnerReference(Topic topic) throws IOException {
return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), topic.getOwner());
@ -284,6 +275,11 @@ public class TopicRepository extends EntityRepository<Topic> {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Topic withHref(URI href) {
return entity.withHref(href);

41
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java
View File

@ -13,8 +13,8 @@
package org.openmetadata.catalog.jdbi3;
import static org.openmetadata.catalog.jdbi3.Relationship.CONTAINS;
import static org.openmetadata.catalog.jdbi3.Relationship.FOLLOWS;
import static org.openmetadata.catalog.jdbi3.Relationship.HAS;
import static org.openmetadata.catalog.jdbi3.Relationship.OWNS;
import com.fasterxml.jackson.core.JsonProcessingException;
@ -102,18 +102,6 @@ public class UserRepository extends EntityRepository<User> {
return setFields(user, fields);
}
@Transaction
public void delete(UUID id) throws IOException {
// Query - mark user as deactivated
User user = markUserAsDeactivated(id);
// Remove relationship membership to teams
dao.relationshipDAO().deleteTo(user.getId().toString(), CONTAINS.ordinal(), "team");
// Remove follows relationship to entities
dao.relationshipDAO().deleteFrom(id.toString(), FOLLOWS.ordinal());
}
@Override
public User setFields(User user, Fields fields) throws IOException {
user.setProfile(fields.contains("profile") ? user.getProfile() : null);
@ -161,7 +149,7 @@ public class UserRepository extends EntityRepository<User> {
/* Add all the teams that user belongs to User entity */
private List<EntityReference> getTeams(User user) throws IOException {
List<String> teamIds = dao.relationshipDAO().findFrom(user.getId().toString(), CONTAINS.ordinal(), "team");
List<String> teamIds = dao.relationshipDAO().findFrom(user.getId().toString(), HAS.ordinal(), "team");
List<EntityReference> teams = new ArrayList<>();
for (String teamId : teamIds) {
teams.add(dao.teamDAO().findEntityReferenceById(UUID.fromString(teamId)));
@ -173,24 +161,10 @@ public class UserRepository extends EntityRepository<User> {
// Query - add team to the user
teams = Optional.ofNullable(teams).orElse(Collections.emptyList());
for (EntityReference team : teams) {
dao.relationshipDAO()
.insert(team.getId().toString(), user.getId().toString(), "team", "user", CONTAINS.ordinal());
dao.relationshipDAO().insert(team.getId().toString(), user.getId().toString(), "team", "user", HAS.ordinal());
}
}
private User markUserAsDeactivated(UUID id) throws IOException {
User user = validateUser(id);
if (Optional.ofNullable(user.getDeactivated()).orElse(false)) {
// User is already deactivated
return user;
}
user.setDeactivated(true);
user.setName("deactivated." + user.getName());
user.setDisplayName("Deactivated " + user.getDisplayName());
dao.userDAO().update(id, JsonUtils.pojoToJson(user));
return user;
}
public static class UserEntityInterface implements EntityInterface<User> {
private final User entity;
@ -299,6 +273,11 @@ public class UserRepository extends EntityRepository<User> {
@Override
public void setOwner(EntityReference owner) {}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public User withHref(URI href) {
return entity.withHref(href);
@ -322,7 +301,7 @@ public class UserRepository extends EntityRepository<User> {
@Override
public void entitySpecificUpdate() throws IOException {
// Update operation can't undelete a user
if (updated.getEntity().getDeactivated() != original.getEntity().getDeactivated()) {
if (updated.getEntity().getDeleted() != original.getEntity().getDeleted()) {
throw new IllegalArgumentException(CatalogExceptionMessage.readOnlyAttribute("User", "deactivated"));
}
updateTeams(original.getEntity(), updated.getEntity());
@ -335,7 +314,7 @@ public class UserRepository extends EntityRepository<User> {
private void updateTeams(User origUser, User updatedUser) throws JsonProcessingException {
// Remove teams from original and add teams from updated
dao.relationshipDAO().deleteTo(origUser.getId().toString(), CONTAINS.ordinal(), "team");
dao.relationshipDAO().deleteTo(origUser.getId().toString(), HAS.ordinal(), "team");
assignTeams(updatedUser, updatedUser.getTeams());
List<EntityReference> origTeams = Optional.ofNullable(origUser.getTeams()).orElse(Collections.emptyList());

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/WebhookRepository.java
View File

@ -281,6 +281,11 @@ public class WebhookRepository extends EntityRepository<Webhook> {
@Override
public void setOwner(EntityReference owner) {}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Webhook withHref(URI href) {
return entity.withHref(href);

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java
View File

@ -390,7 +390,7 @@ public class ChartResource {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "Chart for instance {id} is not found")
})
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) {
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) throws IOException {
dao.delete(UUID.fromString(id));
return Response.ok().build();
}

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java
View File

@ -395,7 +395,7 @@ public class DashboardResource {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "Dashboard for instance {id} is not found")
})
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) {
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) throws IOException {
dao.delete(UUID.fromString(id));
return Response.ok().build();
}

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseResource.java
View File

@ -380,7 +380,7 @@ public class DatabaseResource {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "Database for instance {id} is not found")
})
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) {
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) throws IOException {
dao.delete(UUID.fromString(id));
return Response.ok().build();
}

3
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java
View File

@ -353,7 +353,8 @@ public class TableResource {
public Response delete(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id) {
@Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id)
throws IOException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
dao.delete(UUID.fromString(id));
return Response.ok().build();

3
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/locations/LocationResource.java
View File

@ -404,7 +404,8 @@ public class LocationResource {
public Response delete(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the location", schema = @Schema(type = "string")) @PathParam("id") String id) {
@Parameter(description = "Id of the location", schema = @Schema(type = "string")) @PathParam("id") String id)
throws IOException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
dao.delete(UUID.fromString(id));
return Response.ok().build();

3
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/mlmodels/MlModelResource.java
View File

@ -386,7 +386,8 @@ public class MlModelResource {
public Response delete(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the ML Model", schema = @Schema(type = "string")) @PathParam("id") String id) {
@Parameter(description = "Id of the ML Model", schema = @Schema(type = "string")) @PathParam("id") String id)
throws IOException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
dao.delete(UUID.fromString(id));
return Response.ok().build();

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/IngestionResource.java
View File

@ -401,7 +401,7 @@ public class IngestionResource {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "ingestion for instance {id} is not found")
})
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) {
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) throws IOException {
dao.delete(UUID.fromString(id));
return Response.ok().build();
}

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/pipelines/PipelineResource.java
View File

@ -395,7 +395,7 @@ public class PipelineResource {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "Pipeline for instance {id} is not found")
})
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) {
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) throws IOException {
dao.delete(UUID.fromString(id));
return Response.ok().build();
}

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/policies/PolicyResource.java
View File

@ -346,7 +346,7 @@ public class PolicyResource {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "policy for instance {id} is not found")
})
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) {
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) throws IOException {
dao.delete(UUID.fromString(id));
return Response.ok().build();
}

3
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/dashboard/DashboardServiceResource.java
View File

@ -273,7 +273,8 @@ public class DashboardServiceResource {
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the dashboard service", schema = @Schema(type = "string")) @PathParam("id")
String id) {
String id)
throws IOException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
dao.delete(UUID.fromString(id));
return Response.ok().build();

3
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/database/DatabaseServiceResource.java
View File

@ -269,7 +269,8 @@ public class DatabaseServiceResource {
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the database service", schema = @Schema(type = "string")) @PathParam("id")
String id) {
String id)
throws IOException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
dao.delete(UUID.fromString(id));
return Response.ok().build();

3
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/messaging/MessagingServiceResource.java
View File

@ -279,7 +279,8 @@ public class MessagingServiceResource {
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the messaging service", schema = @Schema(type = "string")) @PathParam("id")
String id) {
String id)
throws IOException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
dao.delete(UUID.fromString(id));
return Response.ok().build();

3
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/pipeline/PipelineServiceResource.java
View File

@ -281,7 +281,8 @@ public class PipelineServiceResource {
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the pipeline service", schema = @Schema(type = "string")) @PathParam("id")
String id) {
String id)
throws IOException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
dao.delete(UUID.fromString(id));
return Response.ok().build();

3
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/storage/StorageServiceResource.java
View File

@ -270,7 +270,8 @@ public class StorageServiceResource {
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the storage service", schema = @Schema(type = "string")) @PathParam("id")
String id) {
String id)
throws IOException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
dao.delete(UUID.fromString(id));
return Response.ok().build();

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/TeamResource.java
View File

@ -337,8 +337,8 @@ public class TeamResource {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "Team for instance {id} is not found")
})
public Response delete(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("id") String id) {
public Response delete(@Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("id") String id)
throws IOException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
dao.delete(UUID.fromString(id));
return Response.ok().build();

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java
View File

@ -390,7 +390,7 @@ public class TopicResource {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "Topic for instance {id} is not found")
})
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) {
public Response delete(@Context UriInfo uriInfo, @PathParam("id") String id) throws IOException {
dao.delete(UUID.fromString(id));
return Response.ok().build();
}

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityInterface.java
View File

@ -65,5 +65,7 @@ public interface EntityInterface<T> {
void setOwner(EntityReference owner);
void setDeleted(boolean flag);
T withHref(URI href);
}

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java
View File

@ -195,7 +195,7 @@ public final class EntityUtil {
if (owner.getType().equalsIgnoreCase("user")) {
User ownerInstance = userDAO.findEntityById(id);
owner.setName(ownerInstance.getName());
if (Optional.ofNullable(ownerInstance.getDeactivated()).orElse(false)) {
if (Optional.ofNullable(ownerInstance.getDeleted()).orElse(false)) {
throw new IllegalArgumentException(CatalogExceptionMessage.deactivatedUser(id));
}
} else if (owner.getType().equalsIgnoreCase("team")) {
@ -360,7 +360,7 @@ public final class EntityUtil {
String followerEntity)
throws IOException {
User user = userDAO.findEntityById(followerId);
if (Optional.ofNullable(user.getDeactivated()).orElse(false)) {
if (Optional.ofNullable(user.getDeleted()).orElse(false)) {
throw new IllegalArgumentException(CatalogExceptionMessage.deactivatedUser(followerId));
}
return dao.insert(

5
catalog-rest-service/src/main/resources/json/schema/entity/bots.json
View File

@ -43,6 +43,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
}
}

5
catalog-rest-service/src/main/resources/json/schema/entity/data/chart.json
View File

@ -140,6 +140,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": ["id", "name", "service"],

5
catalog-rest-service/src/main/resources/json/schema/entity/data/dashboard.json
View File

@ -91,6 +91,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": ["id", "name", "service"],

5
catalog-rest-service/src/main/resources/json/schema/entity/data/database.json
View File

@ -79,6 +79,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": ["name", "service"],

5
catalog-rest-service/src/main/resources/json/schema/entity/data/location.json
View File

@ -106,6 +106,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": [

5
catalog-rest-service/src/main/resources/json/schema/entity/data/metrics.json
View File

@ -70,6 +70,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": ["id", "name", "service"],

5
catalog-rest-service/src/main/resources/json/schema/entity/data/mlmodel.json
View File

@ -274,6 +274,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": ["id", "name", "algorithm"],

5
catalog-rest-service/src/main/resources/json/schema/entity/data/pipeline.json
View File

@ -152,6 +152,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": ["id", "name", "service"],

5
catalog-rest-service/src/main/resources/json/schema/entity/data/report.json
View File

@ -62,6 +62,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": ["id", "name", "service"],

5
catalog-rest-service/src/main/resources/json/schema/entity/data/table.json
View File

@ -565,6 +565,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": [

5
catalog-rest-service/src/main/resources/json/schema/entity/data/topic.json
View File

@ -151,6 +151,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": [

5
catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json
View File

@ -104,6 +104,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": [

5
catalog-rest-service/src/main/resources/json/schema/entity/policies/policy.json
View File

@ -111,6 +111,11 @@
"location": {
"$ref": "../../type/entityReference.json",
"default": null
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": [

5
catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json
View File

@ -93,6 +93,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": [

5
catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json
View File

@ -124,6 +124,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": [

5
catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json
View File

@ -85,6 +85,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": [

5
catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json
View File

@ -77,6 +77,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": [

5
catalog-rest-service/src/main/resources/json/schema/entity/services/storageService.json
View File

@ -47,6 +47,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": [

9
catalog-rest-service/src/main/resources/json/schema/entity/teams/team.json
View File

@ -49,10 +49,6 @@
"description": "Team profile information.",
"$ref": "../../type/profile.json"
},
"deleted" : {
"description": "When true the team has been deleted.",
"type": "boolean"
},
"users" : {
"description": "Users that are part of the team.",
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList",
@ -65,6 +61,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default" : false
}
},
"required" : ["id", "name", "href"],

10
catalog-rest-service/src/main/resources/json/schema/entity/teams/user.json
View File

@ -55,11 +55,6 @@
"type": "string",
"format": "timezone"
},
"deactivated" : {
"description": "When true indicates the user has been deactivated. Users are deactivated instead of deleted.",
"type": "boolean",
"default": false
},
"isBot" : {
"description": "When true indicates a special type of user called Bot.",
"type": "boolean",
@ -89,6 +84,11 @@
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default" : false
}
},
"additionalProperties": false,

5
catalog-rest-service/src/main/resources/json/schema/operations/workflows/ingestion.json
View File

@ -274,6 +274,11 @@
"changeDescription": {
"description" : "Change that led to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": ["name", "service", "connectorConfig", "startDate"],

5
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java
View File

@ -1163,7 +1163,7 @@ public abstract class EntityResourceTest<T> extends CatalogApplicationTest {
UUID ownerId = owner.getId();
List<EntityReference> ownsList;
if (owner.getType().equals(Entity.USER)) {
User user = UserResourceTest.getUser(ownerId, "owns", adminAuthHeaders());
User user = new UserResourceTest().getEntity(ownerId, "owns", adminAuthHeaders());
ownsList = user.getOwns();
} else if (owner.getType().equals(Entity.TEAM)) {
Team team = TeamResourceTest.getTeam(ownerId, "owns", adminAuthHeaders());
@ -1222,9 +1222,6 @@ public abstract class EntityResourceTest<T> extends CatalogApplicationTest {
List<EntityReference> followers = getEntityInterface(getEntity).getFollowers();
TestUtils.validateEntityReference(followers);
TestUtils.existsInEntityReferenceList(followers, userId, false);
// GET .../users/{userId} shows user as following the entity
checkUserFollowing(userId, entityId, false, authHeaders);
return getEntity;
}

8
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java
View File

@ -112,9 +112,9 @@ public class TeamResourceTest extends EntityResourceTest<Team> {
Team team = createAndCheckEntity(create, adminAuthHeaders());
// Make sure the user entity has relationship to the team
user1 = UserResourceTest.getUser(user1.getId(), "teams", authHeaders("test@open-metadata.org"));
user1 = userResourceTest.getEntity(user1.getId(), "teams", authHeaders("test@open-metadata.org"));
assertEquals(team.getId(), user1.getTeams().get(0).getId());
user2 = UserResourceTest.getUser(user2.getId(), "teams", authHeaders("test@open-metadata.org"));
user2 = userResourceTest.getEntity(user2.getId(), "teams", authHeaders("test@open-metadata.org"));
assertEquals(team.getId(), user2.getTeams().get(0).getId());
}
@ -144,10 +144,12 @@ public class TeamResourceTest extends EntityResourceTest<Team> {
List<UUID> users = Collections.singletonList(user1.getId());
CreateTeam create = create(test).withUsers(users);
Team team = createAndCheckEntity(create, adminAuthHeaders());
// Team with users can be deleted - Team -- has --> User relationships are deleted
deleteEntity(team.getId(), adminAuthHeaders());
// Make sure user does not have relationship to this team
User user = UserResourceTest.getUser(user1.getId(), "teams", adminAuthHeaders());
User user = userResourceTest.getEntity(user1.getId(), "teams", adminAuthHeaders());
assertTrue(user.getTeams().isEmpty());
}

45
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/UserResourceTest.java
View File

@ -16,6 +16,7 @@ package org.openmetadata.catalog.resources.teams;
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static javax.ws.rs.core.Response.Status.CREATED;
import static javax.ws.rs.core.Response.Status.FORBIDDEN;
import static javax.ws.rs.core.Response.Status.NOT_FOUND;
import static javax.ws.rs.core.Response.Status.OK;
import static javax.ws.rs.core.Response.Status.UNAUTHORIZED;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -23,7 +24,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.deactivatedUser;
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.readOnlyAttribute;
import static org.openmetadata.catalog.resources.teams.TeamResourceTest.createTeam;
import static org.openmetadata.catalog.security.SecurityUtil.authHeaders;
@ -46,7 +46,6 @@ import java.util.Optional;
import java.util.UUID;
import java.util.function.Predicate;
import javax.json.JsonPatch;
import javax.ws.rs.client.WebTarget;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
@ -194,6 +193,9 @@ public class UserResourceTest extends EntityResourceTest<User> {
List<UUID> teams = List.of(team1.getId(), team2.getId());
List<UUID> team = List.of(team1.getId());
// user0 is part of no teams
// user1 is part of team1
// user2 is part of team1, and team2
CreateUser create = create(test, 0);
User user0 = createAndCheckEntity(create, adminAuthHeaders());
create = create(test, 1).withTeams(team);
@ -238,12 +240,12 @@ public class UserResourceTest extends EntityResourceTest<User> {
// Empty query field .../users?fields=
HttpResponseException exception =
assertThrows(HttpResponseException.class, () -> getUser(user.getId(), "", adminAuthHeaders()));
assertThrows(HttpResponseException.class, () -> getEntity(user.getId(), "", adminAuthHeaders()));
TestUtils.assertResponseContains(exception, BAD_REQUEST, "Invalid field name");
// .../users?fields=invalidField
exception =
assertThrows(HttpResponseException.class, () -> getUser(user.getId(), "invalidField", adminAuthHeaders()));
assertThrows(HttpResponseException.class, () -> getEntity(user.getId(), "invalidField", adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, CatalogExceptionMessage.invalidField("invalidField"));
}
@ -301,7 +303,7 @@ public class UserResourceTest extends EntityResourceTest<User> {
// Ensure user deleted attributed can't be changed using patch
User user = createUser(create(test), adminAuthHeaders());
String userJson = JsonUtils.pojoToJson(user);
user.setDeactivated(true);
user.setDeleted(true);
HttpResponseException exception =
assertThrows(HttpResponseException.class, () -> patchUser(userJson, user, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, readOnlyAttribute("User", "deactivated"));
@ -419,6 +421,7 @@ public class UserResourceTest extends EntityResourceTest<User> {
Table table = tableResourceTest.createEntity(test, 1);
tableResourceTest.addAndCheckFollower(table.getId(), user.getId(), CREATED, 1, adminAuthHeaders());
// Delete user
deleteEntity(user.getId(), adminAuthHeaders());
// Make sure the user is no longer following the table
@ -426,17 +429,12 @@ public class UserResourceTest extends EntityResourceTest<User> {
assertTrue(team.getUsers().isEmpty());
tableResourceTest.checkFollowerDeleted(table.getId(), user.getId(), adminAuthHeaders());
// Get deactivated user and ensure the name and display name has deactivated
User deactivatedUser = getUser(user.getId(), adminAuthHeaders());
assertEquals("deactivated." + user.getName(), deactivatedUser.getName());
assertEquals("Deactivated " + user.getDisplayName(), deactivatedUser.getDisplayName());
// User can no longer follow other entities
HttpResponseException exception =
assertThrows(
HttpResponseException.class,
() -> tableResourceTest.addAndCheckFollower(table.getId(), user.getId(), CREATED, 1, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, deactivatedUser(user.getId()));
assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound("user", user.getId()));
// TODO deactivated user can't be made owner
}
@ -479,8 +477,8 @@ public class UserResourceTest extends EntityResourceTest<User> {
String fields = "profile";
user =
byName
? getUserByName(user.getName(), fields, adminAuthHeaders())
: getUser(user.getId(), fields, adminAuthHeaders());
? getEntityByName(user.getName(), fields, adminAuthHeaders())
: getEntity(user.getId(), fields, adminAuthHeaders());
assertNotNull(user.getProfile());
assertNull(user.getTeams());
@ -488,28 +486,11 @@ public class UserResourceTest extends EntityResourceTest<User> {
fields = "profile, teams";
user =
byName
? getUserByName(user.getName(), fields, adminAuthHeaders())
: getUser(user.getId(), fields, adminAuthHeaders());
? getEntityByName(user.getName(), fields, adminAuthHeaders())
: getEntity(user.getId(), fields, adminAuthHeaders());
assertListNotNull(user.getProfile(), user.getTeams());
}
public static User getUser(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
return getUser(id, null, authHeaders);
}
public static User getUser(UUID id, String fields, Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("users/" + id);
target = fields != null ? target.queryParam("fields", fields) : target;
return TestUtils.get(target, User.class, authHeaders);
}
public static User getUserByName(String name, String fields, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("users/name/" + name);
target = fields != null ? target.queryParam("fields", fields) : target;
return TestUtils.get(target, User.class, authHeaders);
}
@Override
public Object createRequest(String name, String description, String displayName, EntityReference owner) {
return create(name).withDescription(description).withDisplayName(displayName).withProfile(PROFILE);

2
catalog-rest-service/src/test/java/org/openmetadata/catalog/util/TestUtils.java
View File

@ -278,7 +278,7 @@ public final class TestUtils {
UUID userId, UUID entityId, boolean expectedFollowing, Map<String, String> authHeaders)
throws HttpResponseException {
// GET .../users/{userId} shows user as following table
User user = UserResourceTest.getUser(userId, "follows", authHeaders);
User user = new UserResourceTest().getEntity(userId, "follows", authHeaders);
existsInEntityReferenceList(user.getFollows(), entityId, expectedFollowing);
}