yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-08 09:08:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Issue-3596: Provide option to get counts for entities using list APIs by setting limit=0 (#3604)

Browse Source
This commit is contained in:
Sriharsha Chintalapani 2022-03-22 16:45:14 -07:00 committed by GitHub
parent 9ff6b928e5
commit 377d3d6248
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 51 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java
View File

@ -276,24 +276,29 @@ public abstract class EntityRepository<T> {
@Transaction
public final ResultList<T> listAfter(UriInfo uriInfo, Fields fields, ListFilter filter, int limitParam, String after)
throws GeneralSecurityException, IOException, ParseException {
// forward scrolling, if after == null then first page is being asked
List<String> jsons = dao.listAfter(filter, limitParam + 1, after == null ? "" : RestUtil.decodeCursor(after));
List<T> entities = new ArrayList<>();
for (String json : jsons) {
T entity = withHref(uriInfo, setFields(JsonUtils.readValue(json, entityClass), fields));
entities.add(entity);
}
int total = dao.listCount(filter);
List<T> entities = new ArrayList<>();
if (limitParam > 0) {
// forward scrolling, if after == null then first page is being asked
List<String> jsons = dao.listAfter(filter, limitParam + 1, after == null ? "" : RestUtil.decodeCursor(after));
String beforeCursor;
String afterCursor = null;
beforeCursor = after == null ? null : getFullyQualifiedName(entities.get(0));
if (entities.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
entities.remove(limitParam);
afterCursor = getFullyQualifiedName(entities.get(limitParam - 1));
for (String json : jsons) {
T entity = withHref(uriInfo, setFields(JsonUtils.readValue(json, entityClass), fields));
entities.add(entity);
}
String beforeCursor;
String afterCursor = null;
beforeCursor = after == null ? null : getFullyQualifiedName(entities.get(0));
if (entities.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
entities.remove(limitParam);
afterCursor = getFullyQualifiedName(entities.get(limitParam - 1));
}
return getResultList(entities, beforeCursor, afterCursor, total);
} else {
// limit == 0 , return total count of entity.
return getResultList(entities, null, null, total);
}
return getResultList(entities, beforeCursor, afterCursor, total);
}
@Transaction

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/bots/BotsResource.java
View File

@ -87,7 +87,7 @@ public class BotsResource extends EntityResource<Bots, BotsRepository> {
public ResultList<Bots> list(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@DefaultValue("10") @Min(1) @Max(1000000) @QueryParam("limit") int limitParam,
@DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam,
@Parameter(description = "Returns list of bots before this cursor", schema = @Schema(type = "string"))
@QueryParam("before")
String before,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java
View File

@ -129,7 +129,7 @@ public class ChartResource extends EntityResource<Chart, ChartRepository> {
@Parameter(description = "Limit the number charts returned. (1 to 1000000, default = 10)")
@DefaultValue("10")
@QueryParam("limit")
@Min(1)
@Min(0)
@Max(1000000)
int limitParam,
@Parameter(description = "Returns list of charts before this cursor", schema = @Schema(type = "string"))

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java
View File

@ -130,7 +130,7 @@ public class DashboardResource extends EntityResource<Dashboard, DashboardReposi
String serviceParam,
@Parameter(description = "Limit the number dashboards returned. (1 to 1000000, " + "default = 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseResource.java
View File

@ -128,7 +128,7 @@ public class DatabaseResource extends EntityResource<Database, DatabaseRepositor
@Parameter(description = "Limit the number tables returned. (1 to 1000000, default" + " = 10)")
@DefaultValue("10")
@QueryParam("limit")
@Min(1)
@Min(0)
@Max(1000000)
int limitParam,
@Parameter(description = "Returns list of tables before this cursor", schema = @Schema(type = "string"))

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java
View File

@ -153,7 +153,7 @@ public class TableResource extends EntityResource<Table, TableRepository> {
String databaseParam,
@Parameter(description = "Limit the number tables returned. (1 to 1000000, default = " + "10) ")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java
View File

@ -105,7 +105,7 @@ public class WebhookResource extends EntityResource<Webhook, WebhookRepository>
@Context SecurityContext securityContext,
@Parameter(description = "Limit the number webhooks returned. (1 to 1000000, default = " + "10) ")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/feeds/FeedResource.java
View File

@ -135,7 +135,7 @@ public class FeedResource {
@Parameter(
description = "Limit the number of posts sorted by chronological order (1 to 1000000, default = 3)",
schema = @Schema(type = "integer"))
@Min(1)
@Min(0)
@Max(1000000)
@DefaultValue("3")
@QueryParam("limitPosts")

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java
View File

@ -128,7 +128,7 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
String fieldsParam,
@Parameter(description = "Limit the number glossaries returned. (1 to 1000000, " + "default = 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResource.java
View File

@ -146,7 +146,7 @@ public class GlossaryTermResource extends EntityResource<GlossaryTerm, GlossaryT
String fieldsParam,
@Parameter(description = "Limit the number glossary terms returned. (1 to 1000000, " + "default = 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/locations/LocationResource.java
View File

@ -127,7 +127,7 @@ public class LocationResource extends EntityResource<Location, LocationRepositor
String serviceParam,
@Parameter(description = "Limit the number locations returned. " + "(1 to 1000000, default = 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/metrics/MetricsResource.java
View File

@ -98,7 +98,7 @@ public class MetricsResource extends EntityResource<Metrics, MetricsRepository>
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@DefaultValue("10") @Min(1) @Max(1000000) @QueryParam("limit") int limitParam,
@DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam,
@Parameter(description = "Returns list of metrics before this cursor", schema = @Schema(type = "string"))
@QueryParam("before")
String before,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/mlmodels/MlModelResource.java
View File

@ -126,7 +126,7 @@ public class MlModelResource extends EntityResource<MlModel, MlModelRepository>
String fieldsParam,
@Parameter(description = "Limit the number models returned. (1 to 1000000, " + "default = 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/AirflowPipelineResource.java
View File

@ -152,7 +152,7 @@ public class AirflowPipelineResource extends EntityResource<AirflowPipeline, Air
String serviceParam,
@Parameter(description = "Limit the number ingestion returned. (1 to 1000000, " + "default = 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/pipelines/PipelineResource.java
View File

@ -132,7 +132,7 @@ public class PipelineResource extends EntityResource<Pipeline, PipelineRepositor
String serviceParam,
@Parameter(description = "Limit the number pipelines returned. (1 to 1000000, " + "default = 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/policies/PolicyResource.java
View File

@ -137,7 +137,7 @@ public class PolicyResource extends EntityResource<Policy, PolicyRepository> {
String fieldsParam,
@Parameter(description = "Limit the number policies returned. (1 to 1000000, " + "default = 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/dashboard/DashboardServiceResource.java
View File

@ -112,7 +112,7 @@ public class DashboardServiceResource extends EntityResource<DashboardService, D
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@DefaultValue("10") @Min(1) @Max(1000000) @QueryParam("limit") int limitParam,
@DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam,
@Parameter(
description = "Returns list of dashboard services before this cursor",
schema = @Schema(type = "string"))

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/database/DatabaseServiceResource.java
View File

@ -123,7 +123,7 @@ public class DatabaseServiceResource extends EntityResource<DatabaseService, Dat
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@DefaultValue("10") @Min(1) @Max(1000000) @QueryParam("limit") int limitParam,
@DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam,
@Parameter(
description = "Returns list of database services before this cursor",
schema = @Schema(type = "string"))

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/messaging/MessagingServiceResource.java
View File

@ -116,7 +116,7 @@ public class MessagingServiceResource extends EntityResource<MessagingService, M
String fieldsParam,
@Parameter(description = "Limit number services returned. (1 to 1000000, " + "default 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/pipeline/PipelineServiceResource.java
View File

@ -116,7 +116,7 @@ public class PipelineServiceResource extends EntityResource<PipelineService, Pip
String fieldsParam,
@Parameter(description = "Limit number services returned. (1 to 1000000, " + "default 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/storage/StorageServiceResource.java
View File

@ -116,7 +116,7 @@ public class StorageServiceResource extends EntityResource<StorageService, Stora
String fieldsParam,
@Parameter(description = "Limit number of services returned. (1 to 1000000, " + "default 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/RoleResource.java
View File

@ -130,7 +130,7 @@ public class RoleResource extends EntityResource<Role, RoleRepository> {
String fieldsParam,
@Parameter(description = "Limit the number tables returned. (1 to 1000000, default = 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/TeamResource.java
View File

@ -123,7 +123,7 @@ public class TeamResource extends EntityResource<Team, TeamRepository> {
String fieldsParam,
@Parameter(description = "Limit the number tables returned. (1 to 1000000, default = 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/UserResource.java
View File

@ -131,7 +131,7 @@ public class UserResource extends EntityResource<User, UserRepository> {
String teamParam,
@Parameter(description = "Limit the number users returned. (1 to 1000000, default = 10)")
@DefaultValue("10")
@Min(1)
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java
View File

@ -133,7 +133,7 @@ public class TopicResource extends EntityResource<Topic, TopicRepository> {
@Parameter(description = "Limit the number topics returned. (1 to 1000000, default = " + "10)")
@DefaultValue("10")
@QueryParam("limit")
@Min(1)
@Min(0)
@Max(1000000)
int limitParam,
@Parameter(description = "Returns list of topics before this cursor", schema = @Schema(type = "string"))

6
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java
View File

@ -719,12 +719,12 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
assertResponse(
() -> listEntities(null, -1, null, null, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"[query param limit must be greater than or equal to 1]");
"[query param limit must be greater than or equal to 0]");
assertResponse(
() -> listEntities(null, 0, null, null, ADMIN_AUTH_HEADERS),
() -> listEntities(null, -1, null, null, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"[query param limit must be greater than or equal to 1]");
"[query param limit must be greater than or equal to 0]");
assertResponse(
() -> listEntities(null, 1000001, null, null, ADMIN_AUTH_HEADERS),

6
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/feeds/FeedResourceTest.java
View File

@ -366,11 +366,11 @@ public class FeedResourceTest extends CatalogApplicationTest {
thread = threads.getData().get(0);
assertEquals(3, thread.getPosts().size());
// limit 0 is not supported and should throw an exception
// limit <0 is not supported and should throw an exception
assertResponse(
() -> listThreads(null, 0, AUTH_HEADERS),
() -> listThreads(null, -1, AUTH_HEADERS),
BAD_REQUEST,
"[query param limitPosts must be greater than or equal to 1]");
"[query param limitPosts must be greater than or equal to 0]");
// limit greater than total number of posts should return correct response
threads = listThreads(null, 100, AUTH_HEADERS);

7
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java
View File

@ -192,12 +192,7 @@ public class PolicyResourceTest extends EntityResourceTest<Policy, CreatePolicy>
assertResponse(
() -> listPolicies(null, -1, null, null, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"[query param limit must be greater than or equal to 1]");
assertResponse(
() -> listPolicies(null, 0, null, null, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"[query param limit must be greater than or equal to 1]");
"[query param limit must be greater than or equal to 0]");
assertResponse(
() -> listPolicies(null, 1000001, null, null, ADMIN_AUTH_HEADERS),