From 377d3d6248703b8b44b74cf7cf09bb884d0cf17a Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Tue, 22 Mar 2022 16:45:14 -0700 Subject: [PATCH] Issue-3596: Provide option to get counts for entities using list APIs by setting limit=0 (#3604) --- .../catalog/jdbi3/EntityRepository.java | 35 +++++++++++-------- .../catalog/resources/bots/BotsResource.java | 2 +- .../resources/charts/ChartResource.java | 2 +- .../dashboards/DashboardResource.java | 2 +- .../resources/databases/DatabaseResource.java | 2 +- .../resources/databases/TableResource.java | 2 +- .../resources/events/WebhookResource.java | 2 +- .../catalog/resources/feeds/FeedResource.java | 2 +- .../resources/glossary/GlossaryResource.java | 2 +- .../glossary/GlossaryTermResource.java | 2 +- .../resources/locations/LocationResource.java | 2 +- .../resources/metrics/MetricsResource.java | 2 +- .../resources/mlmodels/MlModelResource.java | 2 +- .../operations/AirflowPipelineResource.java | 2 +- .../resources/pipelines/PipelineResource.java | 2 +- .../resources/policies/PolicyResource.java | 2 +- .../dashboard/DashboardServiceResource.java | 2 +- .../database/DatabaseServiceResource.java | 2 +- .../messaging/MessagingServiceResource.java | 2 +- .../pipeline/PipelineServiceResource.java | 2 +- .../storage/StorageServiceResource.java | 2 +- .../catalog/resources/teams/RoleResource.java | 2 +- .../catalog/resources/teams/TeamResource.java | 2 +- .../catalog/resources/teams/UserResource.java | 2 +- .../resources/topics/TopicResource.java | 2 +- .../catalog/resources/EntityResourceTest.java | 6 ++-- .../resources/feeds/FeedResourceTest.java | 6 ++-- .../policies/PolicyResourceTest.java | 7 +--- 28 files changed, 51 insertions(+), 51 deletions(-) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java index cee0a35ba23..a06e72dd8c6 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java @@ -276,24 +276,29 @@ public abstract class EntityRepository { @Transaction public final ResultList listAfter(UriInfo uriInfo, Fields fields, ListFilter filter, int limitParam, String after) throws GeneralSecurityException, IOException, ParseException { - // forward scrolling, if after == null then first page is being asked - List jsons = dao.listAfter(filter, limitParam + 1, after == null ? "" : RestUtil.decodeCursor(after)); - - List entities = new ArrayList<>(); - for (String json : jsons) { - T entity = withHref(uriInfo, setFields(JsonUtils.readValue(json, entityClass), fields)); - entities.add(entity); - } int total = dao.listCount(filter); + List entities = new ArrayList<>(); + if (limitParam > 0) { + // forward scrolling, if after == null then first page is being asked + List jsons = dao.listAfter(filter, limitParam + 1, after == null ? "" : RestUtil.decodeCursor(after)); - String beforeCursor; - String afterCursor = null; - beforeCursor = after == null ? null : getFullyQualifiedName(entities.get(0)); - if (entities.size() > limitParam) { // If extra result exists, then next page exists - return after cursor - entities.remove(limitParam); - afterCursor = getFullyQualifiedName(entities.get(limitParam - 1)); + for (String json : jsons) { + T entity = withHref(uriInfo, setFields(JsonUtils.readValue(json, entityClass), fields)); + entities.add(entity); + } + + String beforeCursor; + String afterCursor = null; + beforeCursor = after == null ? null : getFullyQualifiedName(entities.get(0)); + if (entities.size() > limitParam) { // If extra result exists, then next page exists - return after cursor + entities.remove(limitParam); + afterCursor = getFullyQualifiedName(entities.get(limitParam - 1)); + } + return getResultList(entities, beforeCursor, afterCursor, total); + } else { + // limit == 0 , return total count of entity. + return getResultList(entities, null, null, total); } - return getResultList(entities, beforeCursor, afterCursor, total); } @Transaction diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/bots/BotsResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/bots/BotsResource.java index 16e3de10bfd..2ffd9900e77 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/bots/BotsResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/bots/BotsResource.java @@ -87,7 +87,7 @@ public class BotsResource extends EntityResource { public ResultList list( @Context UriInfo uriInfo, @Context SecurityContext securityContext, - @DefaultValue("10") @Min(1) @Max(1000000) @QueryParam("limit") int limitParam, + @DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam, @Parameter(description = "Returns list of bots before this cursor", schema = @Schema(type = "string")) @QueryParam("before") String before, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java index c011f9bb94e..438a466d234 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java @@ -129,7 +129,7 @@ public class ChartResource extends EntityResource { @Parameter(description = "Limit the number charts returned. (1 to 1000000, default = 10)") @DefaultValue("10") @QueryParam("limit") - @Min(1) + @Min(0) @Max(1000000) int limitParam, @Parameter(description = "Returns list of charts before this cursor", schema = @Schema(type = "string")) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java index aa2918ebdf5..617f19d8ad3 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java @@ -130,7 +130,7 @@ public class DashboardResource extends EntityResource { String databaseParam, @Parameter(description = "Limit the number tables returned. (1 to 1000000, default = " + "10) ") @DefaultValue("10") - @Min(1) + @Min(0) @Max(1000000) @QueryParam("limit") int limitParam, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java index 76518de1d19..dfae5f9ed81 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java @@ -105,7 +105,7 @@ public class WebhookResource extends EntityResource @Context SecurityContext securityContext, @Parameter(description = "Limit the number webhooks returned. (1 to 1000000, default = " + "10) ") @DefaultValue("10") - @Min(1) + @Min(0) @Max(1000000) @QueryParam("limit") int limitParam, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/feeds/FeedResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/feeds/FeedResource.java index 70b3f7af8a4..6c6b149f78a 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/feeds/FeedResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/feeds/FeedResource.java @@ -135,7 +135,7 @@ public class FeedResource { @Parameter( description = "Limit the number of posts sorted by chronological order (1 to 1000000, default = 3)", schema = @Schema(type = "integer")) - @Min(1) + @Min(0) @Max(1000000) @DefaultValue("3") @QueryParam("limitPosts") diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java index dd2672bafed..1c31353368d 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java @@ -128,7 +128,7 @@ public class GlossaryResource extends EntityResource schema = @Schema(type = "string", example = FIELDS)) @QueryParam("fields") String fieldsParam, - @DefaultValue("10") @Min(1) @Max(1000000) @QueryParam("limit") int limitParam, + @DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam, @Parameter(description = "Returns list of metrics before this cursor", schema = @Schema(type = "string")) @QueryParam("before") String before, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/mlmodels/MlModelResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/mlmodels/MlModelResource.java index b70f2d964d1..be30c09461a 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/mlmodels/MlModelResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/mlmodels/MlModelResource.java @@ -126,7 +126,7 @@ public class MlModelResource extends EntityResource String fieldsParam, @Parameter(description = "Limit the number models returned. (1 to 1000000, " + "default = 10)") @DefaultValue("10") - @Min(1) + @Min(0) @Max(1000000) @QueryParam("limit") int limitParam, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/AirflowPipelineResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/AirflowPipelineResource.java index a67891deb83..6831075aa8b 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/AirflowPipelineResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/AirflowPipelineResource.java @@ -152,7 +152,7 @@ public class AirflowPipelineResource extends EntityResource { String fieldsParam, @Parameter(description = "Limit the number policies returned. (1 to 1000000, " + "default = 10)") @DefaultValue("10") - @Min(1) + @Min(0) @Max(1000000) @QueryParam("limit") int limitParam, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/dashboard/DashboardServiceResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/dashboard/DashboardServiceResource.java index 4b9af1a5ec6..47eec26a03e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/dashboard/DashboardServiceResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/dashboard/DashboardServiceResource.java @@ -112,7 +112,7 @@ public class DashboardServiceResource extends EntityResource { String fieldsParam, @Parameter(description = "Limit the number tables returned. (1 to 1000000, default = 10)") @DefaultValue("10") - @Min(1) + @Min(0) @Max(1000000) @QueryParam("limit") int limitParam, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/TeamResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/TeamResource.java index 5ceebf239ee..522ac3c4a6f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/TeamResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/TeamResource.java @@ -123,7 +123,7 @@ public class TeamResource extends EntityResource { String fieldsParam, @Parameter(description = "Limit the number tables returned. (1 to 1000000, default = 10)") @DefaultValue("10") - @Min(1) + @Min(0) @Max(1000000) @QueryParam("limit") int limitParam, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/UserResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/UserResource.java index 6f975a2f899..06dc558321b 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/UserResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/UserResource.java @@ -131,7 +131,7 @@ public class UserResource extends EntityResource { String teamParam, @Parameter(description = "Limit the number users returned. (1 to 1000000, default = 10)") @DefaultValue("10") - @Min(1) + @Min(0) @Max(1000000) @QueryParam("limit") int limitParam, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java index 588bb21a8b5..e49c7c7a5f2 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java @@ -133,7 +133,7 @@ public class TopicResource extends EntityResource { @Parameter(description = "Limit the number topics returned. (1 to 1000000, default = " + "10)") @DefaultValue("10") @QueryParam("limit") - @Min(1) + @Min(0) @Max(1000000) int limitParam, @Parameter(description = "Returns list of topics before this cursor", schema = @Schema(type = "string")) diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java index 6921cd7c773..b2e16e83bf0 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java @@ -719,12 +719,12 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { assertResponse( () -> listEntities(null, -1, null, null, ADMIN_AUTH_HEADERS), BAD_REQUEST, - "[query param limit must be greater than or equal to 1]"); + "[query param limit must be greater than or equal to 0]"); assertResponse( - () -> listEntities(null, 0, null, null, ADMIN_AUTH_HEADERS), + () -> listEntities(null, -1, null, null, ADMIN_AUTH_HEADERS), BAD_REQUEST, - "[query param limit must be greater than or equal to 1]"); + "[query param limit must be greater than or equal to 0]"); assertResponse( () -> listEntities(null, 1000001, null, null, ADMIN_AUTH_HEADERS), diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/feeds/FeedResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/feeds/FeedResourceTest.java index 23adfb7c9b2..51d231296aa 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/feeds/FeedResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/feeds/FeedResourceTest.java @@ -366,11 +366,11 @@ public class FeedResourceTest extends CatalogApplicationTest { thread = threads.getData().get(0); assertEquals(3, thread.getPosts().size()); - // limit 0 is not supported and should throw an exception + // limit <0 is not supported and should throw an exception assertResponse( - () -> listThreads(null, 0, AUTH_HEADERS), + () -> listThreads(null, -1, AUTH_HEADERS), BAD_REQUEST, - "[query param limitPosts must be greater than or equal to 1]"); + "[query param limitPosts must be greater than or equal to 0]"); // limit greater than total number of posts should return correct response threads = listThreads(null, 100, AUTH_HEADERS); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java index 7aa1d37503f..2072ebd42ad 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java @@ -192,12 +192,7 @@ public class PolicyResourceTest extends EntityResourceTest assertResponse( () -> listPolicies(null, -1, null, null, ADMIN_AUTH_HEADERS), BAD_REQUEST, - "[query param limit must be greater than or equal to 1]"); - - assertResponse( - () -> listPolicies(null, 0, null, null, ADMIN_AUTH_HEADERS), - BAD_REQUEST, - "[query param limit must be greater than or equal to 1]"); + "[query param limit must be greater than or equal to 0]"); assertResponse( () -> listPolicies(null, 1000001, null, null, ADMIN_AUTH_HEADERS),