From 269fc9ede5cd0412356af68c24d5528a4444c846 Mon Sep 17 00:00:00 2001 From: sureshms Date: Fri, 26 Nov 2021 14:16:29 -0800 Subject: [PATCH] Fixes #1375 - EventResource malformed entity info --- .../java/org/openmetadata/catalog/Entity.java | 37 ++++++++++------- .../ConstraintViolationExceptionMapper.java | 1 - .../catalog/jdbi3/BotsRepository.java | 3 +- .../catalog/jdbi3/ChangeEventRepository.java | 11 +++-- .../catalog/jdbi3/ChartRepository.java | 3 +- .../catalog/jdbi3/CollectionDAO.java | 24 ++++++++--- .../catalog/jdbi3/DashboardRepository.java | 4 +- .../jdbi3/DashboardServiceRepository.java | 4 +- .../catalog/jdbi3/DatabaseRepository.java | 4 +- .../jdbi3/DatabaseServiceRepository.java | 4 +- .../catalog/jdbi3/DbtModelRepository.java | 2 +- .../catalog/jdbi3/EntityRepository.java | 8 ++-- .../catalog/jdbi3/IngestionRepository.java | 3 +- .../catalog/jdbi3/LocationRepository.java | 4 +- .../jdbi3/MessagingServiceRepository.java | 4 +- .../catalog/jdbi3/MetricsRepository.java | 2 +- .../catalog/jdbi3/MlModelRepository.java | 2 +- .../catalog/jdbi3/PipelineRepository.java | 4 +- .../jdbi3/PipelineServiceRepository.java | 5 +-- .../catalog/jdbi3/PolicyRepository.java | 2 +- .../catalog/jdbi3/ReportRepository.java | 2 +- .../jdbi3/StorageServiceRepository.java | 5 +-- .../catalog/jdbi3/TableRepository.java | 7 ++-- .../catalog/jdbi3/TeamRepository.java | 3 +- .../catalog/jdbi3/TopicRepository.java | 3 +- .../catalog/jdbi3/UserRepository.java | 3 +- .../resources/events/EventResource.java | 18 ++++++--- .../catalog/resources/EntityResourceTest.java | 40 +++++++++++++++++-- 28 files changed, 138 insertions(+), 74 deletions(-) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java index de0d4793776..4e76cda81bd 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java @@ -38,6 +38,7 @@ import java.util.UUID; public final class Entity { private static final Map> DAO_MAP = new HashMap<>(); private static final Map ENTITY_REPOSITORY_MAP = new HashMap<>(); + private static final Map CANONICAL_ENTITY_NAME_MAP = new HashMap<>(); // // Services @@ -83,8 +84,10 @@ public final class Entity { public static void registerEntity(String entity, EntityDAO dao, EntityRepository entityRepository) { - DAO_MAP.put(entity.toLowerCase(Locale.ROOT), dao); + DAO_MAP.put(entity, dao); ENTITY_REPOSITORY_MAP.put(entity, entityRepository); + CANONICAL_ENTITY_NAME_MAP.put(entity.toLowerCase(Locale.ROOT), entity); + System.out.println("Registering entity " + entity); } public static EntityReference getEntityReference(String entity, UUID id) throws IOException { @@ -104,7 +107,7 @@ public final class Entity { } public static EntityReference getEntityReference(Object entity) { - String entityName = entity.getClass().getSimpleName().toLowerCase(Locale.ROOT); + String entityName = getEntityNameFromObject(entity); EntityRepository entityRepository = ENTITY_REPOSITORY_MAP.get(entityName); if (entityRepository == null) { throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(entityName)); @@ -120,7 +123,7 @@ public final class Entity { if (ref == null) { return null; } - String entityName = ref.getType().toLowerCase(Locale.ROOT); + String entityName = ref.getType(); EntityRepository entityRepository = ENTITY_REPOSITORY_MAP.get(entityName); if (entityRepository == null) { throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(entityName)); @@ -133,7 +136,7 @@ public final class Entity { if (entity == null) { return null; } - String entityName = entity.getClass().getSimpleName().toLowerCase(Locale.ROOT); + String entityName = getEntityNameFromObject(entity); EntityRepository entityRepository = ENTITY_REPOSITORY_MAP.get(entityName); if (entityRepository == null) { throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(entityName)); @@ -141,21 +144,27 @@ public final class Entity { return entityRepository.getEntityInterface(entity); } - public static class EntityList { - public static final EntityList EMPTY_LIST = new EntityList(null); - private final List list; + public static String getEntityNameFromObject(Object object) { + return CANONICAL_ENTITY_NAME_MAP.get(object.getClass().getSimpleName().toLowerCase(Locale.ROOT)); + } - public EntityList(String entitiesParam) { + public static class EntityList { + + public static List getEntityList(String name, String entitiesParam) { if (entitiesParam == null) { - list = Collections.emptyList(); - return; + return null; } - list = Arrays.asList(entitiesParam.replaceAll("\\s", "").split(",")); - // TODO validate entity + List list = Arrays.asList(entitiesParam.replaceAll("\\s", "").split(",")); + validateEntities(name, list); + return list; } - public List getList() { - return list; + private static void validateEntities(String name, List list) { + for (String entity : list) { + if (ENTITY_REPOSITORY_MAP.get(entity) == null) { + throw new IllegalArgumentException(String.format("Invalid entity %s in query param %s", entity, name)); + } + } } } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/ConstraintViolationExceptionMapper.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/ConstraintViolationExceptionMapper.java index a4998500907..941bd9a07ee 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/ConstraintViolationExceptionMapper.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/ConstraintViolationExceptionMapper.java @@ -36,7 +36,6 @@ import java.util.stream.Collectors; public class ConstraintViolationExceptionMapper implements ExceptionMapper { @Override public Response toResponse(ConstraintViolationException exception) { - System.out.println(exception); Set> constraintViolations = exception.getConstraintViolations(); List errorMessages = constraintViolations.stream() .map(constraintViolation -> { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java index c585d0fe4df..66623ac977c 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java @@ -37,7 +37,8 @@ public class BotsRepository extends EntityRepository{ private final CollectionDAO dao; public BotsRepository(CollectionDAO dao) { - super(BotsResource.COLLECTION_PATH, Bots.class, dao.botsDAO(), dao, Fields.EMPTY_FIELDS, Fields.EMPTY_FIELDS); + super(BotsResource.COLLECTION_PATH, Entity.BOTS, Bots.class, dao.botsDAO(), dao, Fields.EMPTY_FIELDS, + Fields.EMPTY_FIELDS); this.dao = dao; } public Bots insert(Bots bots) throws JsonProcessingException { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java index be2d3fb7393..47fa24f85fe 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java @@ -17,7 +17,6 @@ package org.openmetadata.catalog.jdbi3; import org.jdbi.v3.sqlobject.transaction.Transaction; -import org.openmetadata.catalog.Entity.EntityList; import org.openmetadata.catalog.resources.events.EventResource.ChangeEventList; import org.openmetadata.catalog.type.ChangeEvent; import org.openmetadata.catalog.util.JsonUtils; @@ -43,13 +42,13 @@ public class ChangeEventRepository { public ChangeEventRepository(CollectionDAO dao) { this.dao = dao; } @Transaction - public ResultList list(Date date, EntityList entityCreatedList, - EntityList entityUpdatedList, EntityList entityDeletedList) throws IOException, + public ResultList list(Date date, List entityCreatedList, + List entityUpdatedList, List entityDeletedList) throws IOException, GeneralSecurityException { List jsons = new ArrayList<>(); - jsons.addAll(dao.changeEventDAO().list(ENTITY_CREATED.value(), entityCreatedList.getList(), date.getTime())); - jsons.addAll(dao.changeEventDAO().list(ENTITY_UPDATED.value(), entityUpdatedList.getList(), date.getTime())); - jsons.addAll(dao.changeEventDAO().list(ENTITY_DELETED.value(), entityDeletedList.getList(), date.getTime())); + jsons.addAll(dao.changeEventDAO().list(ENTITY_CREATED.value(), entityCreatedList, date.getTime())); + jsons.addAll(dao.changeEventDAO().list(ENTITY_UPDATED.value(), entityUpdatedList, date.getTime())); + jsons.addAll(dao.changeEventDAO().list(ENTITY_DELETED.value(), entityDeletedList, date.getTime())); List changeEvents = new ArrayList<>(); for (String json : jsons) { changeEvents.add(JsonUtils.readValue(json, ChangeEvent.class)); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java index baa662ef368..5c77df98b0c 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java @@ -47,7 +47,8 @@ public class ChartRepository extends EntityRepository { private final CollectionDAO dao; public ChartRepository(CollectionDAO dao) { - super(ChartResource.COLLECTION_PATH, Chart.class, dao.chartDAO(), dao, CHART_PATCH_FIELDS, CHART_UPDATE_FIELDS); + super(ChartResource.COLLECTION_PATH, Entity.CHART, Chart.class, dao.chartDAO(), dao, CHART_PATCH_FIELDS, + CHART_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java index 6bf56e523c5..54f56745c59 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java @@ -27,12 +27,12 @@ import org.jdbi.v3.sqlobject.statement.SqlQuery; import org.jdbi.v3.sqlobject.statement.SqlUpdate; import org.openmetadata.catalog.entity.Bots; import org.openmetadata.catalog.entity.data.Chart; -import org.openmetadata.catalog.entity.data.DbtModel; import org.openmetadata.catalog.entity.data.Dashboard; import org.openmetadata.catalog.entity.data.Database; +import org.openmetadata.catalog.entity.data.DbtModel; import org.openmetadata.catalog.entity.data.Location; -import org.openmetadata.catalog.entity.data.MlModel; import org.openmetadata.catalog.entity.data.Metrics; +import org.openmetadata.catalog.entity.data.MlModel; import org.openmetadata.catalog.entity.data.Pipeline; import org.openmetadata.catalog.entity.data.Report; import org.openmetadata.catalog.entity.data.Table; @@ -53,11 +53,11 @@ import org.openmetadata.catalog.jdbi3.DashboardRepository.DashboardEntityInterfa import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface; import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface; import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface; +import org.openmetadata.catalog.jdbi3.DbtModelRepository.DbtModelEntityInterface; import org.openmetadata.catalog.jdbi3.LocationRepository.LocationEntityInterface; import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface; import org.openmetadata.catalog.jdbi3.MetricsRepository.MetricsEntityInterface; import org.openmetadata.catalog.jdbi3.MlModelRepository.MlModelEntityInterface; -import org.openmetadata.catalog.jdbi3.DbtModelRepository.DbtModelEntityInterface; import org.openmetadata.catalog.jdbi3.PipelineRepository.PipelineEntityInterface; import org.openmetadata.catalog.jdbi3.PipelineServiceRepository.PipelineServiceEntityInterface; import org.openmetadata.catalog.jdbi3.PolicyRepository.PolicyEntityInterface; @@ -872,13 +872,27 @@ public interface CollectionDAO { @SqlUpdate("INSERT INTO change_event (json) VALUES (:json)") void insert(@Bind("json") String json); + default List list(String eventType, List entityTypes, long dateTime) { + if (entityTypes == null || entityTypes.isEmpty()) { + return listWithoutEntityFilter(eventType, dateTime); + } + return listWithEntityFilter(eventType, entityTypes, dateTime); + } + @SqlQuery("SELECT json FROM change_event WHERE " + "eventType = :eventType AND " + - "(entityType IN () OR entityType IS NULL) AND " + + "(entityType IN ()) AND " + "dateTime >= :dateTime " + "ORDER BY dateTime DESC") - List list(@Bind("eventType") String eventType, + List listWithEntityFilter(@Bind("eventType") String eventType, @BindList("entityTypes") List entityTypes, @Bind("dateTime") long dateTime); + + @SqlQuery("SELECT json FROM change_event WHERE " + + "eventType = :eventType AND " + + "dateTime >= :dateTime " + + "ORDER BY dateTime DESC") + List listWithoutEntityFilter(@Bind("eventType") String eventType, + @Bind("dateTime") long dateTime); } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java index 3f46e6be7c8..3147a4bf040 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java @@ -51,8 +51,8 @@ public class DashboardRepository extends EntityRepository { private final CollectionDAO dao; public DashboardRepository(CollectionDAO dao) { - super(DashboardResource.COLLECTION_PATH, Dashboard.class, dao.dashboardDAO(), dao, DASHBOARD_PATCH_FIELDS, - DASHBOARD_UPDATE_FIELDS); + super(DashboardResource.COLLECTION_PATH, Entity.DASHBOARD, Dashboard.class, dao.dashboardDAO(), dao, + DASHBOARD_PATCH_FIELDS, DASHBOARD_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java index f0f1e16bf36..d16fc60172e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java @@ -46,8 +46,8 @@ public class DashboardServiceRepository extends EntityRepository { private final CollectionDAO dao; public DatabaseRepository(CollectionDAO dao) { - super(DatabaseResource.COLLECTION_PATH, Database.class, dao.databaseDAO(), dao, DATABASE_PATCH_FIELDS, - DATABASE_UPDATE_FIELDS); + super(DatabaseResource.COLLECTION_PATH, Entity.DATABASE, Database.class, dao.databaseDAO(), dao, + DATABASE_PATCH_FIELDS, DATABASE_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java index 6d853633fc7..c12c44698e8 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java @@ -47,8 +47,8 @@ public class DatabaseServiceRepository extends EntityRepository private final CollectionDAO dao; public DatabaseServiceRepository(CollectionDAO dao) { - super(DatabaseServiceResource.COLLECTION_PATH, DatabaseService.class, dao.dbServiceDAO(), dao, Fields.EMPTY_FIELDS, - Fields.EMPTY_FIELDS); + super(DatabaseServiceResource.COLLECTION_PATH, Entity.DATABASE_SERVICE, DatabaseService.class, + dao.dbServiceDAO(), dao, Fields.EMPTY_FIELDS, Fields.EMPTY_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java index e4f1b243b28..eb71d737464 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java @@ -59,7 +59,7 @@ public class DbtModelRepository extends EntityRepository { private final CollectionDAO dao; public DbtModelRepository(CollectionDAO dao) { - super(DbtModelResource.COLLECTION_PATH, DbtModel.class, dao.dbtModelDAO(), dao, + super(DbtModelResource.COLLECTION_PATH, Entity.DBTMODEL, DbtModel.class, dao.dbtModelDAO(), dao, DBT_MODEL_PATCH_FIELDS, DBT_MODEL_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java index 8eea9d85fd4..82c8ec2e2a3 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java @@ -54,7 +54,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; -import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.UUID; @@ -89,7 +88,8 @@ public abstract class EntityRepository { return new EntityUpdater(original, updated, patchOperation); } - EntityRepository(String collectionPath, Class entityClass, EntityDAO entityDAO, CollectionDAO collectionDAO, + EntityRepository(String collectionPath, String entityName, Class entityClass, EntityDAO entityDAO, + CollectionDAO collectionDAO, Fields patchFields, Fields putFields) { this.collectionPath = collectionPath; this.entityClass = entityClass; @@ -97,7 +97,7 @@ public abstract class EntityRepository { this.daoCollection = collectionDAO; this.patchFields = patchFields; this.putFields = putFields; - this.entityName = entityClass.getSimpleName().toLowerCase(Locale.ROOT); + this.entityName = entityName; Entity.registerEntity(entityName, dao, this); } @@ -230,7 +230,7 @@ public abstract class EntityRepository { entityUpdater.update(); entityUpdater.store(); String change = entityUpdater.fieldsChanged() ? RestUtil.ENTITY_UPDATED : RestUtil.ENTITY_NO_CHANGE; - return new PatchResponse(Status.OK, withHref(uriInfo, updated), change); + return new PatchResponse<>(Status.OK, withHref(uriInfo, updated), change); } @Transaction diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java index f9cfc5c7274..287226cc7ee 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java @@ -47,7 +47,8 @@ public class IngestionRepository extends EntityRepository { private final CollectionDAO dao; public IngestionRepository(CollectionDAO dao) { - super(Entity.INGESTION, Ingestion.class, dao.ingestionDAO(), dao, INGESTION_PATCH_FIELDS, INGESTION_UPDATE_FIELDS); + super(IngestionResource.COLLECTION_PATH, Entity.INGESTION, Ingestion.class, dao.ingestionDAO(), dao, + INGESTION_PATCH_FIELDS, INGESTION_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java index d0e66b27c1b..162f9d3cee8 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java @@ -52,8 +52,8 @@ public class LocationRepository extends EntityRepository { private final CollectionDAO dao; public LocationRepository(CollectionDAO dao) { - super(LocationResource.COLLECTION_PATH, Location.class, dao.locationDAO(), dao, LOCATION_PATCH_FIELDS, - LOCATION_UPDATE_FIELDS); + super(LocationResource.COLLECTION_PATH, Entity.LOCATION, Location.class, dao.locationDAO(), dao, + LOCATION_PATCH_FIELDS, LOCATION_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java index d9143760caf..205bed0735e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java @@ -45,8 +45,8 @@ public class MessagingServiceRepository extends EntityRepository { private final CollectionDAO dao; public MetricsRepository(CollectionDAO dao) { - super(MetricsResource.COLLECTION_PATH, Metrics.class, dao.metricsDAO(), dao, Fields.EMPTY_FIELDS, + super(MetricsResource.COLLECTION_PATH, Entity.METRICS, Metrics.class, dao.metricsDAO(), dao, Fields.EMPTY_FIELDS, METRICS_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java index 31a6e6a5d6b..e3619ca50f9 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java @@ -50,7 +50,7 @@ public class MlModelRepository extends EntityRepository { private final CollectionDAO dao; public MlModelRepository(CollectionDAO dao) { - super(MlModelResource.COLLECTION_PATH, MlModel.class, dao.mlModelDAO(), dao, + super(MlModelResource.COLLECTION_PATH, Entity.MLMODEL, MlModel.class, dao.mlModelDAO(), dao, MODEL_PATCH_FIELDS, MODEL_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java index ef806ee233d..5f1ac2610a9 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java @@ -55,8 +55,8 @@ public class PipelineRepository extends EntityRepository { private final CollectionDAO dao; public PipelineRepository(CollectionDAO dao) { - super(PipelineResource.COLLECTION_PATH, Pipeline.class, dao.pipelineDAO(), dao, PIPELINE_PATCH_FIELDS, - PIPELINE_UPDATE_FIELDS); + super(PipelineResource.COLLECTION_PATH, Entity.PIPELINE, Pipeline.class, dao.pipelineDAO(), dao, + PIPELINE_PATCH_FIELDS, PIPELINE_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java index 3eba2856623..f5e9c5c634e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java @@ -31,7 +31,6 @@ import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.JsonUtils; -import javax.ws.rs.core.UriInfo; import java.io.IOException; import java.net.URI; import java.text.ParseException; @@ -46,8 +45,8 @@ public class PipelineServiceRepository extends EntityRepository private final CollectionDAO dao; public PipelineServiceRepository(CollectionDAO dao) { - super(PipelineServiceResource.COLLECTION_PATH, PipelineService.class, dao.pipelineServiceDAO(), dao, - Fields.EMPTY_FIELDS, Fields.EMPTY_FIELDS); + super(PipelineServiceResource.COLLECTION_PATH, Entity.PIPELINE_SERVICE, PipelineService.class, + dao.pipelineServiceDAO(), dao, Fields.EMPTY_FIELDS, Fields.EMPTY_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java index 610b770febc..459ab363e27 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java @@ -49,7 +49,7 @@ public class PolicyRepository extends EntityRepository { private final CollectionDAO dao; public PolicyRepository(CollectionDAO dao) { - super(PolicyResource.COLLECTION_PATH, Policy.class, dao.policyDAO(), dao, POLICY_PATCH_FIELDS, + super(PolicyResource.COLLECTION_PATH, Entity.POLICY, Policy.class, dao.policyDAO(), dao, POLICY_PATCH_FIELDS, POLICY_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java index 5d44eb9f8e7..90cc66b0290 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java @@ -38,7 +38,7 @@ public class ReportRepository extends EntityRepository { private final CollectionDAO dao; public ReportRepository(CollectionDAO dao) { - super(ReportResource.COLLECTION_PATH, Report.class, dao.reportDAO(), dao, Fields.EMPTY_FIELDS, + super(ReportResource.COLLECTION_PATH, Entity.REPORT, Report.class, dao.reportDAO(), dao, Fields.EMPTY_FIELDS, REPORT_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java index 70f5ee3973b..34d0b90148e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java @@ -27,7 +27,6 @@ import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.util.EntityInterface; import org.openmetadata.catalog.util.JsonUtils; -import javax.ws.rs.core.UriInfo; import java.io.IOException; import java.net.URI; import java.text.ParseException; @@ -42,8 +41,8 @@ public class StorageServiceRepository extends EntityRepository { private final CollectionDAO dao; public StorageServiceRepository(CollectionDAO dao) { - super(StorageServiceResource.COLLECTION_PATH, StorageService.class, dao.storageServiceDAO(), dao, - Fields.EMPTY_FIELDS, Fields.EMPTY_FIELDS); + super(StorageServiceResource.COLLECTION_PATH, Entity.STORAGE_SERVICE, StorageService.class, + dao.storageServiceDAO(), dao, Fields.EMPTY_FIELDS, Fields.EMPTY_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java index cc6dce54633..0be1f692a3a 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.commons.codec.binary.Hex; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.catalog.Entity; -import org.openmetadata.catalog.entity.data.Location; import org.openmetadata.catalog.entity.data.Table; import org.openmetadata.catalog.exception.CatalogExceptionMessage; import org.openmetadata.catalog.exception.EntityNotFoundException; @@ -32,11 +31,11 @@ import org.openmetadata.catalog.type.ColumnProfile; import org.openmetadata.catalog.type.DailyCount; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.JoinedWith; +import org.openmetadata.catalog.type.SQLQuery; import org.openmetadata.catalog.type.TableConstraint; import org.openmetadata.catalog.type.TableData; import org.openmetadata.catalog.type.TableJoins; import org.openmetadata.catalog.type.TableProfile; -import org.openmetadata.catalog.type.SQLQuery; import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.util.EntityInterface; import org.openmetadata.catalog.util.EntityUtil; @@ -47,7 +46,6 @@ import org.openmetadata.common.utils.CommonUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import javax.ws.rs.core.Response.Status; import java.io.IOException; import java.net.URI; @@ -83,7 +81,8 @@ public class TableRepository extends EntityRepository { private final CollectionDAO dao; public TableRepository(CollectionDAO dao) { - super(TableResource.COLLECTION_PATH, Table.class, dao.tableDAO(), dao, TABLE_PATCH_FIELDS, TABLE_UPDATE_FIELDS); + super(TableResource.COLLECTION_PATH, Entity.TABLE, Table.class, dao.tableDAO(), dao, TABLE_PATCH_FIELDS, + TABLE_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java index cb57ed46f57..7a43b185ce1 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java @@ -48,7 +48,8 @@ public class TeamRepository extends EntityRepository { private final CollectionDAO dao; public TeamRepository(CollectionDAO dao) { - super(TeamResource.COLLECTION_PATH, Team.class, dao.teamDAO(), dao, TEAM_PATCH_FIELDS, Fields.EMPTY_FIELDS); + super(TeamResource.COLLECTION_PATH, Entity.TEAM, Team.class, dao.teamDAO(), dao, TEAM_PATCH_FIELDS, + Fields.EMPTY_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java index 2c8adda6853..240f510eebb 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java @@ -50,7 +50,8 @@ public class TopicRepository extends EntityRepository { } public TopicRepository(CollectionDAO dao) { - super(TopicResource.COLLECTION_PATH, Topic.class, dao.topicDAO(), dao, TOPIC_PATCH_FIELDS, TOPIC_UPDATE_FIELDS); + super(TopicResource.COLLECTION_PATH, Entity.TOPIC, Topic.class, dao.topicDAO(), dao, TOPIC_PATCH_FIELDS, + TOPIC_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java index a9fac64cb69..cd129f1ec97 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java @@ -55,7 +55,8 @@ public class UserRepository extends EntityRepository { public UserRepository(CollectionDAO dao) { - super(UserResource.COLLECTION_PATH, User.class, dao.userDAO(), dao, USER_PATCH_FIELDS, USER_UPDATE_FIELDS); + super(UserResource.COLLECTION_PATH, Entity.USER, User.class, dao.userDAO(), dao, USER_PATCH_FIELDS, + USER_UPDATE_FIELDS); this.dao = dao; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java index d3f2e069255..10ffa19da11 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java @@ -82,13 +82,19 @@ public class EventResource { schema = @Schema(implementation = ChangeEvent.class))), @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found")}) public ResultList get(@Context UriInfo uriInfo, - @Parameter(description = "Entities requested for `entityCreated` event", + @Parameter(description = "List of comma separated entities requested for " + + "`entityCreated` event. When null or not set, all entities will be " + + "returned", schema = @Schema(type = "string", example = "table,dashboard,...")) @QueryParam("entityCreated") String entityCreated, - @Parameter(description = "Entities requested for `entityUpdated` event", + @Parameter(description = "List of comma separated entities requested for " + + "`entityUpdated` event. When null or not set, all entities will be " + + "returned", schema = @Schema(type = "string", example = "table,dashboard,...")) @QueryParam("entityUpdated") String entityUpdated, - @Parameter(description = "Entities requested for `entityDeleted` event", + @Parameter(description = "List of comma separated entities requested for " + + "`entityDeleted` event. When null or not set, all entities will be " + + "returned", schema = @Schema(type = "string", example = "table,dashboard,...")) @QueryParam("entityDeleted") String entityDeleted, @Parameter(description = "Events starting from this date time in ISO8601 format", @@ -97,9 +103,9 @@ public class EventResource { @QueryParam("date") String date) throws IOException, GeneralSecurityException, ParseException { Date parsedDate = RestUtil.DATE_TIME_FORMAT.parse(date); - EntityList entityCreatedList = new EntityList(entityCreated); - EntityList entityUpdatedList = new EntityList(entityCreated); - EntityList entityDeletedList = new EntityList(entityCreated); + List entityCreatedList = EntityList.getEntityList("entityCreated", entityCreated); + List entityUpdatedList = EntityList.getEntityList("entityUpdated", entityUpdated); + List entityDeletedList = EntityList.getEntityList("entityDeleted", entityDeleted); return dao.list(parsedDate, entityCreatedList, entityUpdatedList, entityDeletedList); } } \ No newline at end of file diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java index 8642210dd73..59b56e3477b 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java @@ -25,8 +25,8 @@ import org.openmetadata.catalog.jdbi3.PipelineServiceRepository.PipelineServiceE import org.openmetadata.catalog.resources.events.EventResource.ChangeEventList; import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest; import org.openmetadata.catalog.resources.services.MessagingServiceResourceTest; -import org.openmetadata.catalog.resources.tags.TagResourceTest; import org.openmetadata.catalog.resources.services.PipelineServiceResourceTest; +import org.openmetadata.catalog.resources.tags.TagResourceTest; import org.openmetadata.catalog.resources.teams.TeamResourceTest; import org.openmetadata.catalog.resources.teams.UserResourceTest; import org.openmetadata.catalog.type.ChangeDescription; @@ -578,6 +578,27 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { patchEntityAndCheck(entity, origJson, adminAuthHeaders(), MINOR_UPDATE, change); } + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Other tests + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + @Test + public void testInvalidEntityList() { + // Invalid entityCreated list + HttpResponseException exception = assertThrows(HttpResponseException.class, () + -> getChangeEvents("invalidEntity", entityName, null, new Date(), adminAuthHeaders())); + assertResponse(exception, BAD_REQUEST, "Invalid entity invalidEntity in query param entityCreated"); + + // Invalid entityUpdated list + exception = assertThrows(HttpResponseException.class, () + -> getChangeEvents(null, "invalidEntity", entityName, new Date(), adminAuthHeaders())); + assertResponse(exception, BAD_REQUEST, "Invalid entity invalidEntity in query param entityUpdated"); + + // Invalid entityDeleted list + exception = assertThrows(HttpResponseException.class, () + -> getChangeEvents(entityName, null, "invalidEntity", new Date(), adminAuthHeaders())); + assertResponse(exception, BAD_REQUEST, "Invalid entity invalidEntity in query param entityDeleted"); + } + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Common entity functionality for tests @@ -770,6 +791,15 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { Date updateTime, EventType expectedEventType, ChangeDescription expectedChangeDescription, Map authHeaders) throws IOException { + validateChangeEvents(entityInterface, updateTime, expectedEventType, expectedChangeDescription, authHeaders, true); + validateChangeEvents(entityInterface, updateTime, expectedEventType, expectedChangeDescription, authHeaders, false); + } + + private void validateChangeEvents(EntityInterface entityInterface, + Date updateTime, EventType expectedEventType, + ChangeDescription expectedChangeDescription, + Map authHeaders, + boolean withEventFilter) throws IOException { String updatedBy = TestUtils.getPrincipal(authHeaders); ResultList changeEvents; ChangeEvent changeEvent = null; @@ -778,7 +808,11 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { while (iteration < 10) { // Some times change event is not returned on quickly querying with a millisecond // Try multiple times before giving up - changeEvents = getChangeEvents(entityName, entityName, null, updateTime, authHeaders); + if (withEventFilter) { + changeEvents = getChangeEvents(entityName, entityName, null, updateTime, authHeaders); + } else { + changeEvents = getChangeEvents(null, null, null, updateTime, authHeaders); + } assertTrue(changeEvents.getData().size() > 0); for (ChangeEvent event : changeEvents.getData()) { @@ -818,7 +852,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { assertEquals(changeEvent.getPreviousVersion(), 0.1); assertNull(changeEvent.getChangeDescription()); compareEntities(entityInterface.getEntity(), - JsonUtils.readValue((String)changeEvent.getEntity(), entityClass), authHeaders); + JsonUtils.readValue((String) changeEvent.getEntity(), entityClass), authHeaders); } else if (expectedEventType == EventType.ENTITY_UPDATED) { assertNull(changeEvent.getEntity()); assertChangeDescription(expectedChangeDescription, changeEvent.getChangeDescription());