From 076bac5f2d6969933cf5229872ef056e2748a69c Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 6 Sep 2021 13:49:48 -0700 Subject: [PATCH] Fix #379 - Add total number of entities to list APIs (#411) Co-authored-by: sureshms --- .../catalog/CatalogHealthCheck.java | 5 +- .../catalog/jdbi3/ChartRepository.java | 39 +++++++++++---- .../catalog/jdbi3/DashboardRepository.java | 38 +++++++++++---- .../catalog/jdbi3/DatabaseRepository.java | 37 +++++++++++---- .../catalog/jdbi3/TableRepository.java | 47 ++++++++++++++----- .../catalog/jdbi3/TeamRepository.java | 34 +++++++++++--- .../catalog/jdbi3/TopicRepository.java | 42 ++++++++++++----- .../catalog/jdbi3/UserRepository.java | 36 ++++++++++---- .../resources/charts/ChartResource.java | 33 ++++--------- .../dashboards/DashboardResource.java | 33 ++++--------- .../resources/databases/DatabaseResource.java | 29 ++++-------- .../resources/databases/TableResource.java | 29 +++--------- .../catalog/resources/teams/TeamResource.java | 31 ++++-------- .../catalog/resources/teams/UserResource.java | 33 ++++--------- .../resources/topics/TopicResource.java | 36 ++++---------- .../openmetadata/catalog/util/ResultList.java | 41 ++-------------- .../resources/json/schema/type/basic.json | 1 + .../resources/json/schema/type/paging.json | 23 +++++++++ .../openmetadata/catalog/util/TestUtils.java | 2 + 19 files changed, 299 insertions(+), 270 deletions(-) create mode 100644 catalog-rest-service/src/main/resources/json/schema/type/paging.json diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogHealthCheck.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogHealthCheck.java index 07da6989d86..c472b44dd8b 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogHealthCheck.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogHealthCheck.java @@ -17,13 +17,12 @@ package org.openmetadata.catalog; import com.codahale.metrics.health.HealthCheck; -import org.openmetadata.catalog.entity.teams.User; import org.openmetadata.catalog.jdbi3.UserRepository; +import org.openmetadata.catalog.resources.teams.UserResource.UserList; import org.openmetadata.catalog.util.EntityUtil; import org.skife.jdbi.v2.DBI; import java.io.IOException; -import java.util.List; import static org.openmetadata.catalog.resources.teams.UserResource.FIELD_LIST; @@ -39,7 +38,7 @@ public class CatalogHealthCheck extends HealthCheck { @Override protected Result check() throws Exception { try { - List users = userRepository.listAfter(fields, 1, ""); + UserList users = userRepository.listAfter(fields, 1, ""); return Result.healthy(); } catch (IOException e) { return Result.unhealthy(e.getMessage()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java index 892731fe40f..f6448d32c9b 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java @@ -25,6 +25,7 @@ import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServic import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO; import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO; import org.openmetadata.catalog.resources.charts.ChartResource; +import org.openmetadata.catalog.resources.charts.ChartResource.ChartList; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.util.EntityUtil; @@ -79,29 +80,45 @@ public abstract class ChartRepository { @Transaction - public List listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException, + public ChartList listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException, GeneralSecurityException { - // forward scrolling, either because after != null or first page is being asked - List jsons = chartDAO().listAfter(serviceName, limitParam, after == null ? "" : + // forward scrolling, if after == null then first page is being asked being asked + List jsons = chartDAO().listAfter(serviceName, limitParam + 1, after == null ? "" : CipherText.instance().decrypt(after)); List charts = new ArrayList<>(); for (String json : jsons) { charts.add(setFields(JsonUtils.readValue(json, Chart.class), fields)); } - return charts; + int total = chartDAO().listCount(serviceName); + + String beforeCursor, afterCursor = null; + beforeCursor = after == null ? null : charts.get(0).getFullyQualifiedName(); + if (charts.size() > limitParam) { // If extra result exists, then next page exists - return after cursor + charts.remove(limitParam); + afterCursor = charts.get(limitParam - 1).getFullyQualifiedName(); + } + return new ChartList(charts, beforeCursor, afterCursor, total); } @Transaction - public List listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException, + public ChartList listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException, GeneralSecurityException { - // Reverse scrolling - List jsons = chartDAO().listBefore(serviceName, limitParam, CipherText.instance().decrypt(before)); + // Reverse scrolling - Get one extra result used for computing before cursor + List jsons = chartDAO().listBefore(serviceName, limitParam + 1, CipherText.instance().decrypt(before)); List charts = new ArrayList<>(); for (String json : jsons) { charts.add(setFields(JsonUtils.readValue(json, Chart.class), fields)); } - return charts; + int total = chartDAO().listCount(serviceName); + + String beforeCursor = null, afterCursor; + if (charts.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor + charts.remove(0); + beforeCursor = charts.get(0).getFullyQualifiedName(); + } + afterCursor = charts.get(charts.size() - 1).getFullyQualifiedName(); + return new ChartList(charts, beforeCursor, afterCursor, total); } @Transaction @@ -140,7 +157,7 @@ public abstract class ChartRepository { String fqn = getFQN(service, updatedChart); Chart storedDB = JsonUtils.readValue(chartDAO().findByFQN(fqn), Chart.class); if (storedDB == null) { // Chart does not exist. Create a new one - return new PutResponse(Status.CREATED, createInternal(updatedChart, service, newOwner)); + return new PutResponse<>(Status.CREATED, createInternal(updatedChart, service, newOwner)); } // Update the existing chart EntityUtil.populateOwner(userDAO(), teamDAO(), newOwner); // Validate new owner @@ -307,6 +324,10 @@ public abstract class ChartRepository { @SqlQuery("SELECT json FROM chart_entity WHERE id = :id") String findById(@Bind("id") String id); + @SqlQuery("SELECT count(*) FROM chart_entity WHERE " + + "(fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)") + int listCount(@Bind("fqnPrefix") String fqnPrefix); + @SqlQuery( "SELECT json FROM (" + "SELECT fullyQualifiedName, json FROM chart_entity WHERE " + diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java index 56764e2c95b..f1d5e2222c0 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java @@ -28,6 +28,7 @@ import org.openmetadata.catalog.entity.data.Dashboard; import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO; import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO; import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceDAO; +import org.openmetadata.catalog.resources.dashboards.DashboardResource.DashboardList; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.util.EntityUtil; @@ -87,29 +88,45 @@ public abstract class DashboardRepository { @Transaction - public List listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException, + public DashboardList listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException, GeneralSecurityException { - // forward scrolling, either because after != null or first page is being asked - List jsons = dashboardDAO().listAfter(serviceName, limitParam, after == null ? "" : + // forward scrolling, if after == null then first page is being asked being asked + List jsons = dashboardDAO().listAfter(serviceName, limitParam + 1, after == null ? "" : CipherText.instance().decrypt(after)); List dashboards = new ArrayList<>(); for (String json : jsons) { dashboards.add(setFields(JsonUtils.readValue(json, Dashboard.class), fields)); } - return dashboards; + int total = dashboardDAO().listCount(serviceName); + + String beforeCursor, afterCursor = null; + beforeCursor = after == null ? null : dashboards.get(0).getFullyQualifiedName(); + if (dashboards.size() > limitParam) { // If extra result exists, then next page exists - return after cursor + dashboards.remove(limitParam); + afterCursor = dashboards.get(limitParam - 1).getFullyQualifiedName(); + } + return new DashboardList(dashboards, beforeCursor, afterCursor, total); } @Transaction - public List listBefore(Fields fields, String serviceName, int limitParam, String before) + public DashboardList listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException, GeneralSecurityException { - // Reverse scrolling - List jsons = dashboardDAO().listBefore(serviceName, limitParam, CipherText.instance().decrypt(before)); + // Reverse scrolling - Get one extra result used for computing before cursor + List jsons = dashboardDAO().listBefore(serviceName, limitParam + 1, CipherText.instance().decrypt(before)); List dashboards = new ArrayList<>(); for (String json : jsons) { dashboards.add(setFields(JsonUtils.readValue(json, Dashboard.class), fields)); } - return dashboards; + int total = dashboardDAO().listCount(serviceName); + + String beforeCursor = null, afterCursor; + if (dashboards.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor + dashboards.remove(0); + beforeCursor = dashboards.get(0).getFullyQualifiedName(); + } + afterCursor = dashboards.get(dashboards.size() - 1).getFullyQualifiedName(); + return new DashboardList(dashboards, beforeCursor, afterCursor, total); } @Transaction @@ -377,6 +394,10 @@ public abstract class DashboardRepository { @SqlQuery("SELECT json FROM dashboard_entity WHERE fullyQualifiedName = :name") String findByFQN(@Bind("name") String name); + @SqlQuery("SELECT count(*) FROM dashboard_entity WHERE " + + "(fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)") + int listCount(@Bind("fqnPrefix") String fqnPrefix); + @SqlQuery( "SELECT json FROM (" + "SELECT fullyQualifiedName, json FROM dashboard_entity WHERE " + @@ -397,7 +418,6 @@ public abstract class DashboardRepository { List listAfter(@Bind("fqnPrefix") String fqnPrefix, @Bind("limit") int limit, @Bind("after") String after); - @SqlUpdate("DELETE FROM dashboard_entity WHERE id = :id") int delete(@Bind("id") String id); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseRepository.java index 02d12fd40c6..015abc163d0 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseRepository.java @@ -28,6 +28,7 @@ import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO; import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO; import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO; import org.openmetadata.catalog.resources.databases.DatabaseResource; +import org.openmetadata.catalog.resources.databases.DatabaseResource.DatabaseList; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; @@ -93,29 +94,45 @@ public abstract class DatabaseRepository { abstract UsageDAO usageDAO(); @Transaction - public List listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException, + public DatabaseList listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException, GeneralSecurityException { - // forward scrolling, either because after != null or first page is being asked - List jsons = databaseDAO().listAfter(serviceName, limitParam, after == null ? "" : + // forward scrolling, if after == null then first page is being asked being asked + List jsons = databaseDAO().listAfter(serviceName, limitParam + 1, after == null ? "" : CipherText.instance().decrypt(after)); List databases = new ArrayList<>(); for (String json : jsons) { databases.add(setFields(JsonUtils.readValue(json, Database.class), fields)); } - return databases; + int total = databaseDAO().listCount(serviceName); + + String beforeCursor, afterCursor = null; + beforeCursor = after == null ? null : databases.get(0).getFullyQualifiedName(); + if (databases.size() > limitParam) { + databases.remove(limitParam); + afterCursor = databases.get(limitParam - 1).getFullyQualifiedName(); + } + return new DatabaseList(databases, beforeCursor, afterCursor, total); } @Transaction - public List listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException, + public DatabaseList listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException, GeneralSecurityException { - // Reverse scrolling - List jsons = databaseDAO().listBefore(serviceName, limitParam, CipherText.instance().decrypt(before)); + // Reverse scrolling - Get one extra result used for computing before cursor + List jsons = databaseDAO().listBefore(serviceName, limitParam + 1, CipherText.instance().decrypt(before)); List databases = new ArrayList<>(); for (String json : jsons) { databases.add(setFields(JsonUtils.readValue(json, Database.class), fields)); } - return databases; + int total = databaseDAO().listCount(serviceName); + + String beforeCursor = null, afterCursor; + if (databases.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor + databases.remove(0); + beforeCursor = databases.get(0).getFullyQualifiedName(); + } + afterCursor = databases.get(databases.size() - 1).getFullyQualifiedName(); + return new DatabaseList(databases, beforeCursor, afterCursor, total); } @Transaction @@ -310,6 +327,10 @@ public abstract class DatabaseRepository { @SqlQuery("SELECT json FROM database_entity WHERE id = :id") String findById(@Bind("id") String id); + @SqlQuery("SELECT count(*) FROM database_entity WHERE " + + "(fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)") + int listCount(@Bind("fqnPrefix") String fqnPrefix); + @SqlQuery( "SELECT json FROM (" + "SELECT fullyQualifiedName, json FROM database_entity WHERE " + diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java index c72a715eca5..e527805d8a9 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java @@ -17,10 +17,6 @@ package org.openmetadata.catalog.jdbi3; import com.fasterxml.jackson.core.JsonProcessingException; -import org.openmetadata.catalog.type.ColumnJoin; -import org.openmetadata.catalog.type.JoinedWith; -import org.openmetadata.catalog.type.TableData; -import org.openmetadata.catalog.type.TableJoins; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.entity.data.Database; import org.openmetadata.catalog.entity.data.Table; @@ -32,9 +28,14 @@ import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO; import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO; import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO; import org.openmetadata.catalog.resources.databases.TableResource; +import org.openmetadata.catalog.resources.databases.TableResource.TableList; import org.openmetadata.catalog.type.Column; +import org.openmetadata.catalog.type.ColumnJoin; import org.openmetadata.catalog.type.DailyCount; import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.type.JoinedWith; +import org.openmetadata.catalog.type.TableData; +import org.openmetadata.catalog.type.TableJoins; import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; @@ -109,31 +110,47 @@ public abstract class TableRepository { @CreateSqlObject abstract TagDAO tagDAO(); - @Transaction - public List listAfter(Fields fields, String databaseFQN, int limitParam, String after) throws IOException, + @Transaction + public TableList listAfter(Fields fields, String databaseFQN, int limitParam, String after) throws IOException, ParseException, GeneralSecurityException { - // Forward scrolling, either because after != null or first page is being asked - List jsons = tableDAO().listAfter(databaseFQN, limitParam, after == null ? "" : + // forward scrolling, if after == null then first page is being asked being asked + List jsons = tableDAO().listAfter(databaseFQN, limitParam + 1, after == null ? "" : CipherText.instance().decrypt(after)); List
tables = new ArrayList<>(); for (String json : jsons) { tables.add(setFields(JsonUtils.readValue(json, Table.class), fields)); } - return tables; + int total = tableDAO().listCount(databaseFQN); + + String beforeCursor, afterCursor = null; + beforeCursor = after == null ? null : tables.get(0).getFullyQualifiedName(); + if (tables.size() > limitParam) { // If extra result exists, then next page exists - return after cursor + tables.remove(limitParam); + afterCursor = tables.get(limitParam - 1).getFullyQualifiedName(); + } + return new TableList(tables, beforeCursor, afterCursor, total); } @Transaction - public List
listBefore(Fields fields, String databaseFQN, int limitParam, String before) throws IOException, + public TableList listBefore(Fields fields, String databaseFQN, int limitParam, String before) throws IOException, ParseException, GeneralSecurityException { - // Reverse scrolling - List jsons = tableDAO().listBefore(databaseFQN, limitParam, CipherText.instance().decrypt(before)); + // Reverse scrolling - Get one extra result used for computing before cursor + List jsons = tableDAO().listBefore(databaseFQN, limitParam + 1, CipherText.instance().decrypt(before)); List
tables = new ArrayList<>(); for (String json : jsons) { tables.add(setFields(JsonUtils.readValue(json, Table.class), fields)); } - return tables; + int total = tableDAO().listCount(databaseFQN); + + String beforeCursor = null, afterCursor; + if (tables.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor + tables.remove(0); + beforeCursor = tables.get(0).getFullyQualifiedName(); + } + afterCursor = tables.get(tables.size() - 1).getFullyQualifiedName(); + return new TableList(tables, beforeCursor, afterCursor, total); } @Transaction @@ -647,6 +664,10 @@ public abstract class TableRepository { @SqlQuery("SELECT json FROM table_entity WHERE fullyQualifiedName = :tableFQN") String findByFQN(@Bind("tableFQN") String tableFQN); + @SqlQuery("SELECT count(*) FROM table_entity WHERE " + + "(fullyQualifiedName LIKE CONCAT(:databaseFQN, '.%') OR :databaseFQN IS NULL)") + int listCount(@Bind("databaseFQN") String databseFQN); + @SqlQuery( "SELECT json FROM (" + "SELECT fullyQualifiedName, json FROM table_entity WHERE " + diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java index aedeef4cce6..4ab15dd1615 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java @@ -29,6 +29,7 @@ import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO; import org.openmetadata.catalog.jdbi3.TopicRepository.TopicDAO; import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO; import org.openmetadata.catalog.resources.teams.TeamResource; +import org.openmetadata.catalog.resources.teams.TeamResource.TeamList; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; @@ -127,30 +128,47 @@ public abstract class TeamRepository { } @Transaction - public List listAfter(Fields fields, int limitParam, String after) throws IOException, + public TeamList listAfter(Fields fields, int limitParam, String after) throws IOException, ParseException, GeneralSecurityException { // Forward scrolling, either because after != null or first page is being asked - List jsons = teamDAO().listAfter(limitParam, after == null ? "" : + List jsons = teamDAO().listAfter(limitParam + 1, after == null ? "" : CipherText.instance().decrypt(after)); List teams = new ArrayList<>(); for (String json : jsons) { teams.add(setFields(JsonUtils.readValue(json, Team.class), fields)); } - return teams; + + int total = teamDAO().listCount(); + + String beforeCursor, afterCursor = null; + beforeCursor = after == null ? null : teams.get(0).getName(); + if (teams.size() > limitParam) { + teams.remove(limitParam); + afterCursor = teams.get(limitParam - 1).getName(); + } + return new TeamList(teams, beforeCursor, afterCursor, total); } @Transaction - public List listBefore(Fields fields, int limitParam, String before) throws IOException, + public TeamList listBefore(Fields fields, int limitParam, String before) throws IOException, ParseException, GeneralSecurityException { // Reverse scrolling - List jsons = teamDAO().listBefore(limitParam, CipherText.instance().decrypt(before)); + List jsons = teamDAO().listBefore(limitParam + 1, CipherText.instance().decrypt(before)); List teams = new ArrayList<>(); for (String json : jsons) { teams.add(setFields(JsonUtils.readValue(json, Team.class), fields)); } - return teams; + int total = teamDAO().listCount(); + + String beforeCursor = null, afterCursor; + if (teams.size() > limitParam) { + teams.remove(0); + beforeCursor = teams.get(0).getName(); + } + afterCursor = teams.get(teams.size() - 1).getName(); + return new TeamList(teams, beforeCursor, afterCursor, total); } @Transaction @@ -257,6 +275,9 @@ public abstract class TeamRepository { @SqlQuery("SELECT json FROM team_entity where name = :name") String findByName(@Bind("name") String name); + @SqlQuery("SELECT count(*) FROM team_entity") + int listCount(); + @SqlQuery( "SELECT json FROM (" + "SELECT name, json FROM team_entity WHERE " + @@ -272,7 +293,6 @@ public abstract class TeamRepository { "LIMIT :limit") List listAfter(@Bind("limit") int limit, @Bind("after") String after); - @SqlUpdate("DELETE FROM team_entity WHERE id = :teamId") int delete(@Bind("teamId") String teamId); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java index 89ecaf1e78e..8fac1658f92 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java @@ -17,16 +17,15 @@ package org.openmetadata.catalog.jdbi3; import org.openmetadata.catalog.Entity; -import org.openmetadata.catalog.entity.data.Table; import org.openmetadata.catalog.entity.data.Topic; import org.openmetadata.catalog.entity.services.MessagingService; import org.openmetadata.catalog.exception.CatalogExceptionMessage; import org.openmetadata.catalog.exception.EntityNotFoundException; import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceDAO; import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO; -import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO; import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO; import org.openmetadata.catalog.resources.topics.TopicResource; +import org.openmetadata.catalog.resources.topics.TopicResource.TopicList; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.util.EntityUtil; @@ -79,33 +78,46 @@ public abstract class TopicRepository { @CreateSqlObject abstract TagRepository.TagDAO tagDAO(); - @CreateSqlObject - abstract UsageDAO usageDAO(); - @Transaction - public List listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException, + public TopicList listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException, GeneralSecurityException { - // forward scrolling, either because after != null or first page is being asked - List jsons = topicDAO().listAfter(serviceName, limitParam, after == null ? "" : + // forward scrolling, if after == null then first page is being asked being asked + List jsons = topicDAO().listAfter(serviceName, limitParam + 1, after == null ? "" : CipherText.instance().decrypt(after)); List topics = new ArrayList<>(); for (String json : jsons) { topics.add(setFields(JsonUtils.readValue(json, Topic.class), fields)); } - return topics; + int total = topicDAO().listCount(serviceName); + + String beforeCursor, afterCursor = null; + beforeCursor = after == null ? null : topics.get(0).getFullyQualifiedName(); + if (topics.size() > limitParam) { // If extra result exists, then next page exists - return after cursor + topics.remove(limitParam); + afterCursor = topics.get(limitParam - 1).getFullyQualifiedName(); + } + return new TopicList(topics, beforeCursor, afterCursor, total); } @Transaction - public List listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException, + public TopicList listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException, GeneralSecurityException { - // Reverse scrolling - List jsons = topicDAO().listBefore(serviceName, limitParam, CipherText.instance().decrypt(before)); + // Reverse scrolling - Get one extra result used for computing before cursor + List jsons = topicDAO().listBefore(serviceName, limitParam + 1, CipherText.instance().decrypt(before)); List topics = new ArrayList<>(); for (String json : jsons) { topics.add(setFields(JsonUtils.readValue(json, Topic.class), fields)); } - return topics; + int total = topicDAO().listCount(serviceName); + + String beforeCursor = null, afterCursor; + if (topics.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor + topics.remove(0); + beforeCursor = topics.get(0).getFullyQualifiedName(); + } + afterCursor = topics.get(topics.size() - 1).getFullyQualifiedName(); + return new TopicList(topics, beforeCursor, afterCursor, total); } @Transaction @@ -309,6 +321,10 @@ public abstract class TopicRepository { @SqlQuery("SELECT json FROM topic_entity WHERE id = :id") String findById(@Bind("id") String id); + @SqlQuery("SELECT count(*) FROM topic_entity WHERE " + + "(fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)") // Filter by service name + int listCount(@Bind("fqnPrefix") String fqnPrefix); + @SqlQuery( "SELECT json FROM (" + "SELECT fullyQualifiedName, json FROM topic_entity WHERE " + diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java index be00f4bef75..cfe31e49f74 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java @@ -28,6 +28,7 @@ import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO; import org.openmetadata.catalog.jdbi3.TopicRepository.TopicDAO; import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO; import org.openmetadata.catalog.resources.teams.UserResource; +import org.openmetadata.catalog.resources.teams.UserResource.UserList; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; @@ -101,30 +102,46 @@ public abstract class UserRepository { abstract ChartDAO chartDAO(); @Transaction - public List listAfter(Fields fields, int limitParam, String after) throws IOException, + public UserList listAfter(Fields fields, int limitParam, String after) throws IOException, ParseException, GeneralSecurityException { - // Forward scrolling, either because after != null or first page is being asked - List jsons = userDAO().listAfter(limitParam, after == null ? "" : + // forward scrolling, if after == null then first page is being asked being asked + List jsons = userDAO().listAfter(limitParam + 1, after == null ? "" : CipherText.instance().decrypt(after)); List users = new ArrayList<>(); for (String json : jsons) { users.add(setFields(JsonUtils.readValue(json, User.class), fields)); } - return users; + int total = userDAO().listCount(); + + String beforeCursor, afterCursor = null; + beforeCursor = after == null ? null : users.get(0).getName(); + if (users.size() > limitParam) { // If extra result exists, then next page exists - return after cursor + users.remove(limitParam); + afterCursor = users.get(limitParam - 1).getName(); + } + return new UserList(users, beforeCursor, afterCursor, total); } @Transaction - public List listBefore(Fields fields, int limitParam, String before) throws IOException, + public UserList listBefore(Fields fields, int limitParam, String before) throws IOException, ParseException, GeneralSecurityException { - // Reverse scrolling - List jsons = userDAO().listBefore(limitParam, CipherText.instance().decrypt(before)); + // Reverse scrolling - Get one extra result used for computing before cursor + List jsons = userDAO().listBefore(limitParam + 1, CipherText.instance().decrypt(before)); List users = new ArrayList<>(); for (String json : jsons) { users.add(setFields(JsonUtils.readValue(json, User.class), fields)); } - return users; + int total = userDAO().listCount(); + + String beforeCursor = null, afterCursor; + if (users.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor + users.remove(0); + beforeCursor = users.get(0).getName(); + } + afterCursor = users.get(users.size() - 1).getName(); + return new UserList(users, beforeCursor, afterCursor, total); } @Transaction @@ -309,6 +326,9 @@ public abstract class UserRepository { @SqlQuery("SELECT json FROM user_entity") List list(); + @SqlQuery("SELECT count(*) FROM user_entity") + int listCount(); + @SqlQuery( "SELECT json FROM (" + "SELECT name, json FROM user_entity WHERE " + diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java index ed9107bccd8..d1efb0218e7 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java @@ -109,15 +109,15 @@ public class ChartResource { this.authorizer = authorizer; } - static class ChartList extends ResultList { + public static class ChartList extends ResultList { @SuppressWarnings("unused") ChartList() { // Empty constructor needed for deserialization } - ChartList(List data, int limitParam, String beforeCursor, - String afterCursor) throws GeneralSecurityException, UnsupportedEncodingException { - super(data, limitParam, beforeCursor, afterCursor); + public ChartList(List data, String beforeCursor, String afterCursor, int total) + throws GeneralSecurityException, UnsupportedEncodingException { + super(data, beforeCursor, afterCursor, total); } } @@ -159,29 +159,14 @@ public class ChartResource { RestUtil.validateCursors(before, after); Fields fields = new Fields(FIELD_LIST, fieldsParam); - List charts; - String beforeCursor = null, afterCursor = null; - - // For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward - // scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling, - // beforeCursor is not null. Remove the extra entry before returning results. + ChartList charts; if (before != null) { // Reverse paging - charts = dao.listBefore(fields, serviceParam, limitParam + 1, before); // Ask for one extra entry - if (charts.size() > limitParam) { - charts.remove(0); - beforeCursor = charts.get(0).getFullyQualifiedName(); - } - afterCursor = charts.get(charts.size() - 1).getFullyQualifiedName(); + charts = dao.listBefore(fields, serviceParam, limitParam, before); // Ask for one extra entry } else { // Forward paging or first page - charts = dao.listAfter(fields, serviceParam, limitParam + 1, after); - beforeCursor = after == null ? null : charts.get(0).getFullyQualifiedName(); - if (charts.size() > limitParam) { - charts.remove(limitParam); - afterCursor = charts.get(limitParam - 1).getFullyQualifiedName(); - } + charts = dao.listAfter(fields, serviceParam, limitParam, after); } - addHref(uriInfo, charts); - return new ChartList(charts, limitParam, beforeCursor, afterCursor); + addHref(uriInfo, charts.getData()); + return charts; } @GET diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java index a09ff5bb1ce..706dbda2e5e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java @@ -112,15 +112,15 @@ public class DashboardResource { this.authorizer = authorizer; } - static class DashboardList extends ResultList { + public static class DashboardList extends ResultList { @SuppressWarnings("unused") DashboardList() { // Empty constructor needed for deserialization } - DashboardList(List data, int limitParam, String beforeCursor, - String afterCursor) throws GeneralSecurityException, UnsupportedEncodingException { - super(data, limitParam, beforeCursor, afterCursor); + public DashboardList(List data, String beforeCursor, String afterCursor, int total) + throws GeneralSecurityException, UnsupportedEncodingException { + super(data, beforeCursor, afterCursor, total); } } @@ -163,29 +163,14 @@ public class DashboardResource { RestUtil.validateCursors(before, after); Fields fields = new Fields(FIELD_LIST, fieldsParam); - List dashboards; - String beforeCursor = null, afterCursor = null; - - // For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward - // scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling, - // beforeCursor is not null. Remove the extra entry before returning results. + DashboardList dashboards; if (before != null) { // Reverse paging - dashboards = dao.listBefore(fields, serviceParam, limitParam + 1, before); // Ask for one extra entry - if (dashboards.size() > limitParam) { - dashboards.remove(0); - beforeCursor = dashboards.get(0).getFullyQualifiedName(); - } - afterCursor = dashboards.get(dashboards.size() - 1).getFullyQualifiedName(); + dashboards = dao.listBefore(fields, serviceParam, limitParam, before); // Ask for one extra entry } else { // Forward paging or first page - dashboards = dao.listAfter(fields, serviceParam, limitParam + 1, after); - beforeCursor = after == null ? null : dashboards.get(0).getFullyQualifiedName(); - if (dashboards.size() > limitParam) { - dashboards.remove(limitParam); - afterCursor = dashboards.get(limitParam - 1).getFullyQualifiedName(); - } + dashboards = dao.listAfter(fields, serviceParam, limitParam, after); } - addHref(uriInfo, dashboards); - return new DashboardList(dashboards, limitParam, beforeCursor, afterCursor); + addHref(uriInfo, dashboards.getData()); + return dashboards; } @GET diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseResource.java index 6426d060b35..0ad1e499b8e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseResource.java @@ -111,12 +111,12 @@ public class DatabaseResource { this.authorizer = authorizer; } - static class DatabaseList extends ResultList { + public static class DatabaseList extends ResultList { @SuppressWarnings("unused") // Empty constructor needed for deserialization DatabaseList() {} - DatabaseList(List data, int limitParam, String beforeCursor, - String afterCursor) throws GeneralSecurityException, UnsupportedEncodingException { - super(data, limitParam, beforeCursor, afterCursor); + public DatabaseList(List data, String beforeCursor, String afterCursor, int total) + throws GeneralSecurityException, UnsupportedEncodingException { + super(data, beforeCursor, afterCursor, total); } } @@ -158,29 +158,18 @@ public class DatabaseResource { RestUtil.validateCursors(before, after); Fields fields = new Fields(FIELD_LIST, fieldsParam); - List databases; - String beforeCursor = null, afterCursor = null; + DatabaseList databases; // For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward // scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling, // beforeCursor is not null. Remove the extra entry before returning results. if (before != null) { // Reverse paging - databases = dao.listBefore(fields, serviceParam, limitParam + 1, before); // Ask for one extra entry - if (databases.size() > limitParam) { - databases.remove(0); - beforeCursor = databases.get(0).getFullyQualifiedName(); - } - afterCursor = databases.get(databases.size() - 1).getFullyQualifiedName(); + databases = dao.listBefore(fields, serviceParam, limitParam, before); // Ask for one extra entry } else { // Forward paging or first page - databases = dao.listAfter(fields, serviceParam, limitParam + 1, after); - beforeCursor = after == null ? null : databases.get(0).getFullyQualifiedName(); - if (databases.size() > limitParam) { - databases.remove(limitParam); - afterCursor = databases.get(limitParam - 1).getFullyQualifiedName(); - } + databases = dao.listAfter(fields, serviceParam, limitParam, after); } - addHref(uriInfo, databases); - return new DatabaseList(databases, limitParam, beforeCursor, afterCursor); + addHref(uriInfo, databases.getData()); + return databases; } @GET diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java index 2a5f03cce19..057086e5dda 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java @@ -109,9 +109,9 @@ public class TableResource { @SuppressWarnings("unused") /* Required for tests */ public TableList() {} - public TableList(List
data, int limitParam, String beforeCursor, String afterCursor) + public TableList(List
data, String beforeCursor, String afterCursor, int total) throws GeneralSecurityException, UnsupportedEncodingException { - super(data, limitParam, beforeCursor, afterCursor); + super(data, beforeCursor, afterCursor, total); } } @@ -153,29 +153,14 @@ public class TableResource { RestUtil.validateCursors(before, after); Fields fields = new Fields(FIELD_LIST, fieldsParam); - List
tables; - String beforeCursor = null, afterCursor = null; - - // For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward - // scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling, - // beforeCursor is not null. Remove the extra entry before returning results. + TableList tables; if (before != null) { // Reverse paging - tables = dao.listBefore(fields, databaseParam, limitParam + 1, before); // Ask for one extra entry - if (tables.size() > limitParam) { - tables.remove(0); - beforeCursor = tables.get(0).getFullyQualifiedName(); - } - afterCursor = tables.get(tables.size() - 1).getFullyQualifiedName(); + tables = dao.listBefore(fields, databaseParam, limitParam, before); } else { // Forward paging or first page - tables = dao.listAfter(fields, databaseParam, limitParam + 1, after); - beforeCursor = after == null ? null : tables.get(0).getFullyQualifiedName(); - if (tables.size() > limitParam) { - tables.remove(limitParam); - afterCursor = tables.get(limitParam - 1).getFullyQualifiedName(); - } + tables = dao.listAfter(fields, databaseParam, limitParam, after); } - tables.forEach(t -> addHref(uriInfo, t)); - return new TableList(tables, limitParam, beforeCursor, afterCursor); + tables.getData().forEach(t -> addHref(uriInfo, t)); + return tables; } @GET diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/TeamResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/TeamResource.java index 76a7efb9e6b..576202889a8 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/TeamResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/TeamResource.java @@ -97,13 +97,13 @@ public class TeamResource { this.authorizer = authorizer; } - static class TeamList extends ResultList { + public static class TeamList extends ResultList { @SuppressWarnings("unused") /* Required for tests */ TeamList() {} - TeamList(List teams, int limitParam, String beforeCursor, String afterCursor) + public TeamList(List teams, String beforeCursor, String afterCursor, int total) throws GeneralSecurityException, UnsupportedEncodingException { - super(teams, limitParam, beforeCursor, afterCursor); + super(teams, beforeCursor, afterCursor, total); } } @@ -140,30 +140,15 @@ public class TeamResource { @QueryParam("after") String after) throws IOException, GeneralSecurityException, ParseException { RestUtil.validateCursors(before, after); EntityUtil.Fields fields = new EntityUtil.Fields(FIELD_LIST, fieldsParam); - List teams; - String beforeCursor = null, afterCursor = null; - // For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward - // scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling, - // beforeCursor is not null. Remove the extra entry before returning results. + TeamList teams; if (before != null) { // Reverse paging - teams = dao.listBefore(fields, limitParam + 1, before); // Ask for one extra entry - if (teams.size() > limitParam) { - teams.remove(0); - beforeCursor = teams.get(0).getName(); - } - afterCursor = teams.get(teams.size() - 1).getName(); + teams = dao.listBefore(fields, limitParam, before); // Ask for one extra entry } else { // Forward paging or first page - teams = dao.listAfter(fields, limitParam + 1, after); - beforeCursor = after == null ? null : teams.get(0).getName(); - if (teams.size() > limitParam) { - teams.remove(limitParam); - afterCursor = teams.get(limitParam - 1).getName(); - } + teams = dao.listAfter(fields, limitParam, after); } - teams.forEach(team -> addHref(uriInfo, team)); - LOG.info("Returning {} teams", teams.size()); - return new TeamList(teams, limitParam, beforeCursor, afterCursor); + teams.getData().forEach(team -> addHref(uriInfo, team)); + return teams; } @GET diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/UserResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/UserResource.java index 58cb771c10d..7d887250c3d 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/UserResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/UserResource.java @@ -100,13 +100,13 @@ public class UserResource { this.authorizer = authorizer; } - static class UserList extends ResultList { + public static class UserList extends ResultList { @SuppressWarnings("unused") // Used for deserialization - UserList() {} + public UserList() {} - UserList(List users, int limitParam, String beforeCursor, String afterCursor) + public UserList(List users, String beforeCursor, String afterCursor, int total) throws GeneralSecurityException, UnsupportedEncodingException { - super(users, limitParam, beforeCursor, afterCursor); + super(users, beforeCursor, afterCursor, total); } } @@ -144,29 +144,14 @@ public class UserResource { RestUtil.validateCursors(before, after); Fields fields = new Fields(FIELD_LIST, fieldsParam); - List users; - String beforeCursor = null, afterCursor = null; - - // For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward - // scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling, - // beforeCursor is not null. Remove the extra entry before returning results. + UserList users; if (before != null) { // Reverse paging - users = dao.listBefore(fields, limitParam + 1, before); // Ask for one extra entry - if (users.size() > limitParam) { - users.remove(0); - beforeCursor = users.get(0).getName(); - } - afterCursor = users.get(users.size() - 1).getName(); + users = dao.listBefore(fields, limitParam, before); } else { // Forward paging or first page - users = dao.listAfter(fields, limitParam + 1, after); - beforeCursor = after == null ? null : users.get(0).getName(); - if (users.size() > limitParam) { - users.remove(limitParam); - afterCursor = users.get(limitParam - 1).getName(); - } + users = dao.listAfter(fields, limitParam, after); } - Optional.ofNullable(users).orElse(Collections.emptyList()).forEach(u -> addHref(uriInfo, u)); - return new UserList(users, limitParam, beforeCursor, afterCursor); + Optional.ofNullable(users.getData()).orElse(Collections.emptyList()).forEach(u -> addHref(uriInfo, u)); + return users; } @GET diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java index 279657a44bb..e44d02018a6 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java @@ -28,7 +28,6 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import org.openmetadata.catalog.api.data.CreateTopic; import org.openmetadata.catalog.entity.data.Dashboard; -import org.openmetadata.catalog.entity.data.Table; import org.openmetadata.catalog.entity.data.Topic; import org.openmetadata.catalog.jdbi3.TopicRepository; import org.openmetadata.catalog.resources.Collection; @@ -109,15 +108,15 @@ public class TopicResource { this.authorizer = authorizer; } - static class TopicList extends ResultList { + public static class TopicList extends ResultList { @SuppressWarnings("unused") - TopicList() { + public TopicList() { // Empty constructor needed for deserialization } - TopicList(List data, int limitParam, String beforeCursor, - String afterCursor) throws GeneralSecurityException, UnsupportedEncodingException { - super(data, limitParam, beforeCursor, afterCursor); + public TopicList(List data, String beforeCursor, String afterCursor, int total) + throws GeneralSecurityException, UnsupportedEncodingException { + super(data, beforeCursor, afterCursor, total); } } @@ -159,29 +158,14 @@ public class TopicResource { RestUtil.validateCursors(before, after); Fields fields = new Fields(FIELD_LIST, fieldsParam); - List topics; - String beforeCursor = null, afterCursor = null; - - // For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward - // scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling, - // beforeCursor is not null. Remove the extra entry before returning results. + TopicList topics; if (before != null) { // Reverse paging - topics = dao.listBefore(fields, serviceParam, limitParam + 1, before); // Ask for one extra entry - if (topics.size() > limitParam) { - topics.remove(0); - beforeCursor = topics.get(0).getFullyQualifiedName(); - } - afterCursor = topics.get(topics.size() - 1).getFullyQualifiedName(); + topics = dao.listBefore(fields, serviceParam, limitParam, before); // Ask for one extra entry } else { // Forward paging or first page - topics = dao.listAfter(fields, serviceParam, limitParam + 1, after); - beforeCursor = after == null ? null : topics.get(0).getFullyQualifiedName(); - if (topics.size() > limitParam) { - topics.remove(limitParam); - afterCursor = topics.get(limitParam - 1).getFullyQualifiedName(); - } + topics = dao.listAfter(fields, serviceParam, limitParam, after); } - addHref(uriInfo, topics); - return new TopicList(topics, limitParam, beforeCursor, afterCursor); + addHref(uriInfo, topics.getData()); + return topics; } @GET diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/ResultList.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/ResultList.java index 8a7d5a00601..9f358cf035d 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/ResultList.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/ResultList.java @@ -19,6 +19,7 @@ package org.openmetadata.catalog.util; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import org.openmetadata.catalog.type.Paging; import org.openmetadata.common.utils.CipherText; import javax.validation.constraints.NotNull; @@ -39,38 +40,6 @@ import java.util.List; @JsonPropertyOrder({"data"}) public class ResultList { - public static class Paging { - @JsonProperty("before") - @NotNull - private String before; - - @JsonProperty("after") - @NotNull - private String after; - - @JsonProperty("before") - public String getBefore() { - return before; - } - - @JsonProperty("before") - public Paging setBefore(String before) { - this.before = before; - return this; - } - - @JsonProperty("after") - public String getAfter() { - return after; - } - - @JsonProperty("after") - public Paging setAfter(String after) { - this.after = after; - return this; - } - } - @JsonProperty("data") @NotNull private List data; @@ -139,12 +108,11 @@ public class ResultList { * -------- BACKWARD SCROLLING ENDS ------------- * */ - public ResultList(List data, int limit, String beforeCursor, String afterCursor) throws GeneralSecurityException, + public ResultList(List data, String beforeCursor, String afterCursor, int total) throws GeneralSecurityException, UnsupportedEncodingException { this.data = data; - paging = new Paging(); - paging.before = CipherText.instance().encrypt(beforeCursor); - paging.after = CipherText.instance().encrypt(afterCursor); + paging = new Paging().withBefore(CipherText.instance().encrypt(beforeCursor)) + .withAfter(CipherText.instance().encrypt(afterCursor)).withTotal(total); } @JsonProperty("data") @@ -167,5 +135,4 @@ public class ResultList { this.paging = paging; return this; } - } diff --git a/catalog-rest-service/src/main/resources/json/schema/type/basic.json b/catalog-rest-service/src/main/resources/json/schema/type/basic.json index dbd030002a9..24b2ec40db5 100644 --- a/catalog-rest-service/src/main/resources/json/schema/type/basic.json +++ b/catalog-rest-service/src/main/resources/json/schema/type/basic.json @@ -34,6 +34,7 @@ }, "timeInterval": { "type": "object", + "description": "Time interval in unixTimeMillis.", "javaType": "org.openmetadata.catalog.type.TimeInterval", "properties": { "start": { diff --git a/catalog-rest-service/src/main/resources/json/schema/type/paging.json b/catalog-rest-service/src/main/resources/json/schema/type/paging.json new file mode 100644 index 00000000000..42e2175889b --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/type/paging.json @@ -0,0 +1,23 @@ +{ + "$id": "https://open-metadata.org/schema/type/paging.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Paging", + "description": "Type used for cursor based pagination information in GET list responses.", + "type": "object", + "javaType": "org.openmetadata.catalog.type.Paging", + "properties": { + "before": { + "description": "Before cursor used for getting the previous page (see API pagination for details).", + "type": "string" + }, + "after": { + "description": "After cursor used for getting the next page (see API pagination for details).", + "type": "string" + }, + "total": { + "description": "Total number of entries available to page through.", + "type" : "integer" + } + }, + "required": ["total"] +} \ No newline at end of file diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/util/TestUtils.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/util/TestUtils.java index 4e554ce88cb..fbd7e91c693 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/util/TestUtils.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/util/TestUtils.java @@ -121,6 +121,8 @@ public final class TestUtils { for (int i = 0; i < actual.getData().size(); i++) { assertEquals(allEntities.get(offset + i), actual.getData().get(i)); } + // Ensure total count returned in paging is correct + assertEquals(allEntities.size(), actual.getPaging().getTotal()); } public static void post(WebTarget target, Map headers) throws HttpResponseException {