Fix #379 - Add total number of entities to list APIs (#411)

Co-authored-by: sureshms <suresh@getcollate.io>
This commit is contained in:
Suresh Srinivas 2021-09-06 13:49:48 -07:00 committed by GitHub
parent 27a02fa379
commit 076bac5f2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 299 additions and 270 deletions

View File

@ -17,13 +17,12 @@
package org.openmetadata.catalog; package org.openmetadata.catalog;
import com.codahale.metrics.health.HealthCheck; import com.codahale.metrics.health.HealthCheck;
import org.openmetadata.catalog.entity.teams.User;
import org.openmetadata.catalog.jdbi3.UserRepository; import org.openmetadata.catalog.jdbi3.UserRepository;
import org.openmetadata.catalog.resources.teams.UserResource.UserList;
import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil;
import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.DBI;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import static org.openmetadata.catalog.resources.teams.UserResource.FIELD_LIST; import static org.openmetadata.catalog.resources.teams.UserResource.FIELD_LIST;
@ -39,7 +38,7 @@ public class CatalogHealthCheck extends HealthCheck {
@Override @Override
protected Result check() throws Exception { protected Result check() throws Exception {
try { try {
List<User> users = userRepository.listAfter(fields, 1, ""); UserList users = userRepository.listAfter(fields, 1, "");
return Result.healthy(); return Result.healthy();
} catch (IOException e) { } catch (IOException e) {
return Result.unhealthy(e.getMessage()); return Result.unhealthy(e.getMessage());

View File

@ -25,6 +25,7 @@ import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServic
import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO; import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO; import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
import org.openmetadata.catalog.resources.charts.ChartResource; import org.openmetadata.catalog.resources.charts.ChartResource;
import org.openmetadata.catalog.resources.charts.ChartResource.ChartList;
import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil;
@ -79,29 +80,45 @@ public abstract class ChartRepository {
@Transaction @Transaction
public List<Chart> listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException, public ChartList listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException,
GeneralSecurityException { GeneralSecurityException {
// forward scrolling, either because after != null or first page is being asked // forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = chartDAO().listAfter(serviceName, limitParam, after == null ? "" : List<String> jsons = chartDAO().listAfter(serviceName, limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after)); CipherText.instance().decrypt(after));
List<Chart> charts = new ArrayList<>(); List<Chart> charts = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
charts.add(setFields(JsonUtils.readValue(json, Chart.class), fields)); charts.add(setFields(JsonUtils.readValue(json, Chart.class), fields));
} }
return charts; int total = chartDAO().listCount(serviceName);
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : charts.get(0).getFullyQualifiedName();
if (charts.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
charts.remove(limitParam);
afterCursor = charts.get(limitParam - 1).getFullyQualifiedName();
}
return new ChartList(charts, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
public List<Chart> listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException, public ChartList listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException,
GeneralSecurityException { GeneralSecurityException {
// Reverse scrolling // Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = chartDAO().listBefore(serviceName, limitParam, CipherText.instance().decrypt(before)); List<String> jsons = chartDAO().listBefore(serviceName, limitParam + 1, CipherText.instance().decrypt(before));
List<Chart> charts = new ArrayList<>(); List<Chart> charts = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
charts.add(setFields(JsonUtils.readValue(json, Chart.class), fields)); charts.add(setFields(JsonUtils.readValue(json, Chart.class), fields));
} }
return charts; int total = chartDAO().listCount(serviceName);
String beforeCursor = null, afterCursor;
if (charts.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
charts.remove(0);
beforeCursor = charts.get(0).getFullyQualifiedName();
}
afterCursor = charts.get(charts.size() - 1).getFullyQualifiedName();
return new ChartList(charts, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
@ -140,7 +157,7 @@ public abstract class ChartRepository {
String fqn = getFQN(service, updatedChart); String fqn = getFQN(service, updatedChart);
Chart storedDB = JsonUtils.readValue(chartDAO().findByFQN(fqn), Chart.class); Chart storedDB = JsonUtils.readValue(chartDAO().findByFQN(fqn), Chart.class);
if (storedDB == null) { // Chart does not exist. Create a new one if (storedDB == null) { // Chart does not exist. Create a new one
return new PutResponse<Chart>(Status.CREATED, createInternal(updatedChart, service, newOwner)); return new PutResponse<>(Status.CREATED, createInternal(updatedChart, service, newOwner));
} }
// Update the existing chart // Update the existing chart
EntityUtil.populateOwner(userDAO(), teamDAO(), newOwner); // Validate new owner EntityUtil.populateOwner(userDAO(), teamDAO(), newOwner); // Validate new owner
@ -307,6 +324,10 @@ public abstract class ChartRepository {
@SqlQuery("SELECT json FROM chart_entity WHERE id = :id") @SqlQuery("SELECT json FROM chart_entity WHERE id = :id")
String findById(@Bind("id") String id); String findById(@Bind("id") String id);
@SqlQuery("SELECT count(*) FROM chart_entity WHERE " +
"(fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)")
int listCount(@Bind("fqnPrefix") String fqnPrefix);
@SqlQuery( @SqlQuery(
"SELECT json FROM (" + "SELECT json FROM (" +
"SELECT fullyQualifiedName, json FROM chart_entity WHERE " + "SELECT fullyQualifiedName, json FROM chart_entity WHERE " +

View File

@ -28,6 +28,7 @@ import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO; import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO;
import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO; import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO;
import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceDAO; import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceDAO;
import org.openmetadata.catalog.resources.dashboards.DashboardResource.DashboardList;
import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil;
@ -87,29 +88,45 @@ public abstract class DashboardRepository {
@Transaction @Transaction
public List<Dashboard> listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException, public DashboardList listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException,
GeneralSecurityException { GeneralSecurityException {
// forward scrolling, either because after != null or first page is being asked // forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = dashboardDAO().listAfter(serviceName, limitParam, after == null ? "" : List<String> jsons = dashboardDAO().listAfter(serviceName, limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after)); CipherText.instance().decrypt(after));
List<Dashboard> dashboards = new ArrayList<>(); List<Dashboard> dashboards = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
dashboards.add(setFields(JsonUtils.readValue(json, Dashboard.class), fields)); dashboards.add(setFields(JsonUtils.readValue(json, Dashboard.class), fields));
} }
return dashboards; int total = dashboardDAO().listCount(serviceName);
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : dashboards.get(0).getFullyQualifiedName();
if (dashboards.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
dashboards.remove(limitParam);
afterCursor = dashboards.get(limitParam - 1).getFullyQualifiedName();
}
return new DashboardList(dashboards, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
public List<Dashboard> listBefore(Fields fields, String serviceName, int limitParam, String before) public DashboardList listBefore(Fields fields, String serviceName, int limitParam, String before)
throws IOException, GeneralSecurityException { throws IOException, GeneralSecurityException {
// Reverse scrolling // Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = dashboardDAO().listBefore(serviceName, limitParam, CipherText.instance().decrypt(before)); List<String> jsons = dashboardDAO().listBefore(serviceName, limitParam + 1, CipherText.instance().decrypt(before));
List<Dashboard> dashboards = new ArrayList<>(); List<Dashboard> dashboards = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
dashboards.add(setFields(JsonUtils.readValue(json, Dashboard.class), fields)); dashboards.add(setFields(JsonUtils.readValue(json, Dashboard.class), fields));
} }
return dashboards; int total = dashboardDAO().listCount(serviceName);
String beforeCursor = null, afterCursor;
if (dashboards.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
dashboards.remove(0);
beforeCursor = dashboards.get(0).getFullyQualifiedName();
}
afterCursor = dashboards.get(dashboards.size() - 1).getFullyQualifiedName();
return new DashboardList(dashboards, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
@ -377,6 +394,10 @@ public abstract class DashboardRepository {
@SqlQuery("SELECT json FROM dashboard_entity WHERE fullyQualifiedName = :name") @SqlQuery("SELECT json FROM dashboard_entity WHERE fullyQualifiedName = :name")
String findByFQN(@Bind("name") String name); String findByFQN(@Bind("name") String name);
@SqlQuery("SELECT count(*) FROM dashboard_entity WHERE " +
"(fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)")
int listCount(@Bind("fqnPrefix") String fqnPrefix);
@SqlQuery( @SqlQuery(
"SELECT json FROM (" + "SELECT json FROM (" +
"SELECT fullyQualifiedName, json FROM dashboard_entity WHERE " + "SELECT fullyQualifiedName, json FROM dashboard_entity WHERE " +
@ -397,7 +418,6 @@ public abstract class DashboardRepository {
List<String> listAfter(@Bind("fqnPrefix") String fqnPrefix, @Bind("limit") int limit, List<String> listAfter(@Bind("fqnPrefix") String fqnPrefix, @Bind("limit") int limit,
@Bind("after") String after); @Bind("after") String after);
@SqlUpdate("DELETE FROM dashboard_entity WHERE id = :id") @SqlUpdate("DELETE FROM dashboard_entity WHERE id = :id")
int delete(@Bind("id") String id); int delete(@Bind("id") String id);
} }

View File

@ -28,6 +28,7 @@ import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO; import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO;
import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO; import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
import org.openmetadata.catalog.resources.databases.DatabaseResource; import org.openmetadata.catalog.resources.databases.DatabaseResource;
import org.openmetadata.catalog.resources.databases.DatabaseResource.DatabaseList;
import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.EntityUtil.Fields;
@ -93,29 +94,45 @@ public abstract class DatabaseRepository {
abstract UsageDAO usageDAO(); abstract UsageDAO usageDAO();
@Transaction @Transaction
public List<Database> listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException, public DatabaseList listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException,
GeneralSecurityException { GeneralSecurityException {
// forward scrolling, either because after != null or first page is being asked // forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = databaseDAO().listAfter(serviceName, limitParam, after == null ? "" : List<String> jsons = databaseDAO().listAfter(serviceName, limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after)); CipherText.instance().decrypt(after));
List<Database> databases = new ArrayList<>(); List<Database> databases = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
databases.add(setFields(JsonUtils.readValue(json, Database.class), fields)); databases.add(setFields(JsonUtils.readValue(json, Database.class), fields));
} }
return databases; int total = databaseDAO().listCount(serviceName);
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : databases.get(0).getFullyQualifiedName();
if (databases.size() > limitParam) {
databases.remove(limitParam);
afterCursor = databases.get(limitParam - 1).getFullyQualifiedName();
}
return new DatabaseList(databases, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
public List<Database> listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException, public DatabaseList listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException,
GeneralSecurityException { GeneralSecurityException {
// Reverse scrolling // Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = databaseDAO().listBefore(serviceName, limitParam, CipherText.instance().decrypt(before)); List<String> jsons = databaseDAO().listBefore(serviceName, limitParam + 1, CipherText.instance().decrypt(before));
List<Database> databases = new ArrayList<>(); List<Database> databases = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
databases.add(setFields(JsonUtils.readValue(json, Database.class), fields)); databases.add(setFields(JsonUtils.readValue(json, Database.class), fields));
} }
return databases; int total = databaseDAO().listCount(serviceName);
String beforeCursor = null, afterCursor;
if (databases.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
databases.remove(0);
beforeCursor = databases.get(0).getFullyQualifiedName();
}
afterCursor = databases.get(databases.size() - 1).getFullyQualifiedName();
return new DatabaseList(databases, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
@ -310,6 +327,10 @@ public abstract class DatabaseRepository {
@SqlQuery("SELECT json FROM database_entity WHERE id = :id") @SqlQuery("SELECT json FROM database_entity WHERE id = :id")
String findById(@Bind("id") String id); String findById(@Bind("id") String id);
@SqlQuery("SELECT count(*) FROM database_entity WHERE " +
"(fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)")
int listCount(@Bind("fqnPrefix") String fqnPrefix);
@SqlQuery( @SqlQuery(
"SELECT json FROM (" + "SELECT json FROM (" +
"SELECT fullyQualifiedName, json FROM database_entity WHERE " + "SELECT fullyQualifiedName, json FROM database_entity WHERE " +

View File

@ -17,10 +17,6 @@
package org.openmetadata.catalog.jdbi3; package org.openmetadata.catalog.jdbi3;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import org.openmetadata.catalog.type.ColumnJoin;
import org.openmetadata.catalog.type.JoinedWith;
import org.openmetadata.catalog.type.TableData;
import org.openmetadata.catalog.type.TableJoins;
import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Database; import org.openmetadata.catalog.entity.data.Database;
import org.openmetadata.catalog.entity.data.Table; import org.openmetadata.catalog.entity.data.Table;
@ -32,9 +28,14 @@ import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO; import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO;
import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO; import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
import org.openmetadata.catalog.resources.databases.TableResource; import org.openmetadata.catalog.resources.databases.TableResource;
import org.openmetadata.catalog.resources.databases.TableResource.TableList;
import org.openmetadata.catalog.type.Column; import org.openmetadata.catalog.type.Column;
import org.openmetadata.catalog.type.ColumnJoin;
import org.openmetadata.catalog.type.DailyCount; import org.openmetadata.catalog.type.DailyCount;
import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.JoinedWith;
import org.openmetadata.catalog.type.TableData;
import org.openmetadata.catalog.type.TableJoins;
import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.EntityUtil.Fields;
@ -109,31 +110,47 @@ public abstract class TableRepository {
@CreateSqlObject @CreateSqlObject
abstract TagDAO tagDAO(); abstract TagDAO tagDAO();
@Transaction @Transaction
public List<Table> listAfter(Fields fields, String databaseFQN, int limitParam, String after) throws IOException, public TableList listAfter(Fields fields, String databaseFQN, int limitParam, String after) throws IOException,
ParseException, GeneralSecurityException { ParseException, GeneralSecurityException {
// Forward scrolling, either because after != null or first page is being asked // forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = tableDAO().listAfter(databaseFQN, limitParam, after == null ? "" : List<String> jsons = tableDAO().listAfter(databaseFQN, limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after)); CipherText.instance().decrypt(after));
List<Table> tables = new ArrayList<>(); List<Table> tables = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
tables.add(setFields(JsonUtils.readValue(json, Table.class), fields)); tables.add(setFields(JsonUtils.readValue(json, Table.class), fields));
} }
return tables; int total = tableDAO().listCount(databaseFQN);
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : tables.get(0).getFullyQualifiedName();
if (tables.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
tables.remove(limitParam);
afterCursor = tables.get(limitParam - 1).getFullyQualifiedName();
}
return new TableList(tables, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
public List<Table> listBefore(Fields fields, String databaseFQN, int limitParam, String before) throws IOException, public TableList listBefore(Fields fields, String databaseFQN, int limitParam, String before) throws IOException,
ParseException, GeneralSecurityException { ParseException, GeneralSecurityException {
// Reverse scrolling // Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = tableDAO().listBefore(databaseFQN, limitParam, CipherText.instance().decrypt(before)); List<String> jsons = tableDAO().listBefore(databaseFQN, limitParam + 1, CipherText.instance().decrypt(before));
List<Table> tables = new ArrayList<>(); List<Table> tables = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
tables.add(setFields(JsonUtils.readValue(json, Table.class), fields)); tables.add(setFields(JsonUtils.readValue(json, Table.class), fields));
} }
return tables; int total = tableDAO().listCount(databaseFQN);
String beforeCursor = null, afterCursor;
if (tables.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
tables.remove(0);
beforeCursor = tables.get(0).getFullyQualifiedName();
}
afterCursor = tables.get(tables.size() - 1).getFullyQualifiedName();
return new TableList(tables, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
@ -647,6 +664,10 @@ public abstract class TableRepository {
@SqlQuery("SELECT json FROM table_entity WHERE fullyQualifiedName = :tableFQN") @SqlQuery("SELECT json FROM table_entity WHERE fullyQualifiedName = :tableFQN")
String findByFQN(@Bind("tableFQN") String tableFQN); String findByFQN(@Bind("tableFQN") String tableFQN);
@SqlQuery("SELECT count(*) FROM table_entity WHERE " +
"(fullyQualifiedName LIKE CONCAT(:databaseFQN, '.%') OR :databaseFQN IS NULL)")
int listCount(@Bind("databaseFQN") String databseFQN);
@SqlQuery( @SqlQuery(
"SELECT json FROM (" + "SELECT json FROM (" +
"SELECT fullyQualifiedName, json FROM table_entity WHERE " + "SELECT fullyQualifiedName, json FROM table_entity WHERE " +

View File

@ -29,6 +29,7 @@ import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
import org.openmetadata.catalog.jdbi3.TopicRepository.TopicDAO; import org.openmetadata.catalog.jdbi3.TopicRepository.TopicDAO;
import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO; import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO;
import org.openmetadata.catalog.resources.teams.TeamResource; import org.openmetadata.catalog.resources.teams.TeamResource;
import org.openmetadata.catalog.resources.teams.TeamResource.TeamList;
import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.EntityUtil.Fields;
@ -127,30 +128,47 @@ public abstract class TeamRepository {
} }
@Transaction @Transaction
public List<Team> listAfter(Fields fields, int limitParam, String after) throws IOException, public TeamList listAfter(Fields fields, int limitParam, String after) throws IOException,
ParseException, GeneralSecurityException { ParseException, GeneralSecurityException {
// Forward scrolling, either because after != null or first page is being asked // Forward scrolling, either because after != null or first page is being asked
List<String> jsons = teamDAO().listAfter(limitParam, after == null ? "" : List<String> jsons = teamDAO().listAfter(limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after)); CipherText.instance().decrypt(after));
List<Team> teams = new ArrayList<>(); List<Team> teams = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
teams.add(setFields(JsonUtils.readValue(json, Team.class), fields)); teams.add(setFields(JsonUtils.readValue(json, Team.class), fields));
} }
return teams;
int total = teamDAO().listCount();
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : teams.get(0).getName();
if (teams.size() > limitParam) {
teams.remove(limitParam);
afterCursor = teams.get(limitParam - 1).getName();
}
return new TeamList(teams, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
public List<Team> listBefore(Fields fields, int limitParam, String before) throws IOException, public TeamList listBefore(Fields fields, int limitParam, String before) throws IOException,
ParseException, GeneralSecurityException { ParseException, GeneralSecurityException {
// Reverse scrolling // Reverse scrolling
List<String> jsons = teamDAO().listBefore(limitParam, CipherText.instance().decrypt(before)); List<String> jsons = teamDAO().listBefore(limitParam + 1, CipherText.instance().decrypt(before));
List<Team> teams = new ArrayList<>(); List<Team> teams = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
teams.add(setFields(JsonUtils.readValue(json, Team.class), fields)); teams.add(setFields(JsonUtils.readValue(json, Team.class), fields));
} }
return teams; int total = teamDAO().listCount();
String beforeCursor = null, afterCursor;
if (teams.size() > limitParam) {
teams.remove(0);
beforeCursor = teams.get(0).getName();
}
afterCursor = teams.get(teams.size() - 1).getName();
return new TeamList(teams, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
@ -257,6 +275,9 @@ public abstract class TeamRepository {
@SqlQuery("SELECT json FROM team_entity where name = :name") @SqlQuery("SELECT json FROM team_entity where name = :name")
String findByName(@Bind("name") String name); String findByName(@Bind("name") String name);
@SqlQuery("SELECT count(*) FROM team_entity")
int listCount();
@SqlQuery( @SqlQuery(
"SELECT json FROM (" + "SELECT json FROM (" +
"SELECT name, json FROM team_entity WHERE " + "SELECT name, json FROM team_entity WHERE " +
@ -272,7 +293,6 @@ public abstract class TeamRepository {
"LIMIT :limit") "LIMIT :limit")
List<String> listAfter(@Bind("limit") int limit, @Bind("after") String after); List<String> listAfter(@Bind("limit") int limit, @Bind("after") String after);
@SqlUpdate("DELETE FROM team_entity WHERE id = :teamId") @SqlUpdate("DELETE FROM team_entity WHERE id = :teamId")
int delete(@Bind("teamId") String teamId); int delete(@Bind("teamId") String teamId);

View File

@ -17,16 +17,15 @@
package org.openmetadata.catalog.jdbi3; package org.openmetadata.catalog.jdbi3;
import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.entity.data.Topic; import org.openmetadata.catalog.entity.data.Topic;
import org.openmetadata.catalog.entity.services.MessagingService; import org.openmetadata.catalog.entity.services.MessagingService;
import org.openmetadata.catalog.exception.CatalogExceptionMessage; import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException; import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceDAO; import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceDAO;
import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO; import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO;
import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO; import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
import org.openmetadata.catalog.resources.topics.TopicResource; import org.openmetadata.catalog.resources.topics.TopicResource;
import org.openmetadata.catalog.resources.topics.TopicResource.TopicList;
import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil;
@ -79,33 +78,46 @@ public abstract class TopicRepository {
@CreateSqlObject @CreateSqlObject
abstract TagRepository.TagDAO tagDAO(); abstract TagRepository.TagDAO tagDAO();
@CreateSqlObject
abstract UsageDAO usageDAO();
@Transaction @Transaction
public List<Topic> listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException, public TopicList listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException,
GeneralSecurityException { GeneralSecurityException {
// forward scrolling, either because after != null or first page is being asked // forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = topicDAO().listAfter(serviceName, limitParam, after == null ? "" : List<String> jsons = topicDAO().listAfter(serviceName, limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after)); CipherText.instance().decrypt(after));
List<Topic> topics = new ArrayList<>(); List<Topic> topics = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
topics.add(setFields(JsonUtils.readValue(json, Topic.class), fields)); topics.add(setFields(JsonUtils.readValue(json, Topic.class), fields));
} }
return topics; int total = topicDAO().listCount(serviceName);
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : topics.get(0).getFullyQualifiedName();
if (topics.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
topics.remove(limitParam);
afterCursor = topics.get(limitParam - 1).getFullyQualifiedName();
}
return new TopicList(topics, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
public List<Topic> listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException, public TopicList listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException,
GeneralSecurityException { GeneralSecurityException {
// Reverse scrolling // Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = topicDAO().listBefore(serviceName, limitParam, CipherText.instance().decrypt(before)); List<String> jsons = topicDAO().listBefore(serviceName, limitParam + 1, CipherText.instance().decrypt(before));
List<Topic> topics = new ArrayList<>(); List<Topic> topics = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
topics.add(setFields(JsonUtils.readValue(json, Topic.class), fields)); topics.add(setFields(JsonUtils.readValue(json, Topic.class), fields));
} }
return topics; int total = topicDAO().listCount(serviceName);
String beforeCursor = null, afterCursor;
if (topics.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
topics.remove(0);
beforeCursor = topics.get(0).getFullyQualifiedName();
}
afterCursor = topics.get(topics.size() - 1).getFullyQualifiedName();
return new TopicList(topics, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
@ -309,6 +321,10 @@ public abstract class TopicRepository {
@SqlQuery("SELECT json FROM topic_entity WHERE id = :id") @SqlQuery("SELECT json FROM topic_entity WHERE id = :id")
String findById(@Bind("id") String id); String findById(@Bind("id") String id);
@SqlQuery("SELECT count(*) FROM topic_entity WHERE " +
"(fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)") // Filter by service name
int listCount(@Bind("fqnPrefix") String fqnPrefix);
@SqlQuery( @SqlQuery(
"SELECT json FROM (" + "SELECT json FROM (" +
"SELECT fullyQualifiedName, json FROM topic_entity WHERE " + "SELECT fullyQualifiedName, json FROM topic_entity WHERE " +

View File

@ -28,6 +28,7 @@ import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.TopicRepository.TopicDAO; import org.openmetadata.catalog.jdbi3.TopicRepository.TopicDAO;
import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO; import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO;
import org.openmetadata.catalog.resources.teams.UserResource; import org.openmetadata.catalog.resources.teams.UserResource;
import org.openmetadata.catalog.resources.teams.UserResource.UserList;
import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.EntityUtil.Fields;
@ -101,30 +102,46 @@ public abstract class UserRepository {
abstract ChartDAO chartDAO(); abstract ChartDAO chartDAO();
@Transaction @Transaction
public List<User> listAfter(Fields fields, int limitParam, String after) throws IOException, public UserList listAfter(Fields fields, int limitParam, String after) throws IOException,
ParseException, GeneralSecurityException { ParseException, GeneralSecurityException {
// Forward scrolling, either because after != null or first page is being asked // forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = userDAO().listAfter(limitParam, after == null ? "" : List<String> jsons = userDAO().listAfter(limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after)); CipherText.instance().decrypt(after));
List<User> users = new ArrayList<>(); List<User> users = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
users.add(setFields(JsonUtils.readValue(json, User.class), fields)); users.add(setFields(JsonUtils.readValue(json, User.class), fields));
} }
return users; int total = userDAO().listCount();
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : users.get(0).getName();
if (users.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
users.remove(limitParam);
afterCursor = users.get(limitParam - 1).getName();
}
return new UserList(users, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
public List<User> listBefore(Fields fields, int limitParam, String before) throws IOException, public UserList listBefore(Fields fields, int limitParam, String before) throws IOException,
ParseException, GeneralSecurityException { ParseException, GeneralSecurityException {
// Reverse scrolling // Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = userDAO().listBefore(limitParam, CipherText.instance().decrypt(before)); List<String> jsons = userDAO().listBefore(limitParam + 1, CipherText.instance().decrypt(before));
List<User> users = new ArrayList<>(); List<User> users = new ArrayList<>();
for (String json : jsons) { for (String json : jsons) {
users.add(setFields(JsonUtils.readValue(json, User.class), fields)); users.add(setFields(JsonUtils.readValue(json, User.class), fields));
} }
return users; int total = userDAO().listCount();
String beforeCursor = null, afterCursor;
if (users.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
users.remove(0);
beforeCursor = users.get(0).getName();
}
afterCursor = users.get(users.size() - 1).getName();
return new UserList(users, beforeCursor, afterCursor, total);
} }
@Transaction @Transaction
@ -309,6 +326,9 @@ public abstract class UserRepository {
@SqlQuery("SELECT json FROM user_entity") @SqlQuery("SELECT json FROM user_entity")
List<String> list(); List<String> list();
@SqlQuery("SELECT count(*) FROM user_entity")
int listCount();
@SqlQuery( @SqlQuery(
"SELECT json FROM (" + "SELECT json FROM (" +
"SELECT name, json FROM user_entity WHERE " + "SELECT name, json FROM user_entity WHERE " +

View File

@ -109,15 +109,15 @@ public class ChartResource {
this.authorizer = authorizer; this.authorizer = authorizer;
} }
static class ChartList extends ResultList<Chart> { public static class ChartList extends ResultList<Chart> {
@SuppressWarnings("unused") @SuppressWarnings("unused")
ChartList() { ChartList() {
// Empty constructor needed for deserialization // Empty constructor needed for deserialization
} }
ChartList(List<Chart> data, int limitParam, String beforeCursor, public ChartList(List<Chart> data, String beforeCursor, String afterCursor, int total)
String afterCursor) throws GeneralSecurityException, UnsupportedEncodingException { throws GeneralSecurityException, UnsupportedEncodingException {
super(data, limitParam, beforeCursor, afterCursor); super(data, beforeCursor, afterCursor, total);
} }
} }
@ -159,29 +159,14 @@ public class ChartResource {
RestUtil.validateCursors(before, after); RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam); Fields fields = new Fields(FIELD_LIST, fieldsParam);
List<Chart> charts; ChartList charts;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
if (before != null) { // Reverse paging if (before != null) { // Reverse paging
charts = dao.listBefore(fields, serviceParam, limitParam + 1, before); // Ask for one extra entry charts = dao.listBefore(fields, serviceParam, limitParam, before); // Ask for one extra entry
if (charts.size() > limitParam) {
charts.remove(0);
beforeCursor = charts.get(0).getFullyQualifiedName();
}
afterCursor = charts.get(charts.size() - 1).getFullyQualifiedName();
} else { // Forward paging or first page } else { // Forward paging or first page
charts = dao.listAfter(fields, serviceParam, limitParam + 1, after); charts = dao.listAfter(fields, serviceParam, limitParam, after);
beforeCursor = after == null ? null : charts.get(0).getFullyQualifiedName();
if (charts.size() > limitParam) {
charts.remove(limitParam);
afterCursor = charts.get(limitParam - 1).getFullyQualifiedName();
}
} }
addHref(uriInfo, charts); addHref(uriInfo, charts.getData());
return new ChartList(charts, limitParam, beforeCursor, afterCursor); return charts;
} }
@GET @GET

View File

@ -112,15 +112,15 @@ public class DashboardResource {
this.authorizer = authorizer; this.authorizer = authorizer;
} }
static class DashboardList extends ResultList<Dashboard> { public static class DashboardList extends ResultList<Dashboard> {
@SuppressWarnings("unused") @SuppressWarnings("unused")
DashboardList() { DashboardList() {
// Empty constructor needed for deserialization // Empty constructor needed for deserialization
} }
DashboardList(List<Dashboard> data, int limitParam, String beforeCursor, public DashboardList(List<Dashboard> data, String beforeCursor, String afterCursor, int total)
String afterCursor) throws GeneralSecurityException, UnsupportedEncodingException { throws GeneralSecurityException, UnsupportedEncodingException {
super(data, limitParam, beforeCursor, afterCursor); super(data, beforeCursor, afterCursor, total);
} }
} }
@ -163,29 +163,14 @@ public class DashboardResource {
RestUtil.validateCursors(before, after); RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam); Fields fields = new Fields(FIELD_LIST, fieldsParam);
List<Dashboard> dashboards; DashboardList dashboards;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
if (before != null) { // Reverse paging if (before != null) { // Reverse paging
dashboards = dao.listBefore(fields, serviceParam, limitParam + 1, before); // Ask for one extra entry dashboards = dao.listBefore(fields, serviceParam, limitParam, before); // Ask for one extra entry
if (dashboards.size() > limitParam) {
dashboards.remove(0);
beforeCursor = dashboards.get(0).getFullyQualifiedName();
}
afterCursor = dashboards.get(dashboards.size() - 1).getFullyQualifiedName();
} else { // Forward paging or first page } else { // Forward paging or first page
dashboards = dao.listAfter(fields, serviceParam, limitParam + 1, after); dashboards = dao.listAfter(fields, serviceParam, limitParam, after);
beforeCursor = after == null ? null : dashboards.get(0).getFullyQualifiedName();
if (dashboards.size() > limitParam) {
dashboards.remove(limitParam);
afterCursor = dashboards.get(limitParam - 1).getFullyQualifiedName();
}
} }
addHref(uriInfo, dashboards); addHref(uriInfo, dashboards.getData());
return new DashboardList(dashboards, limitParam, beforeCursor, afterCursor); return dashboards;
} }
@GET @GET

View File

@ -111,12 +111,12 @@ public class DatabaseResource {
this.authorizer = authorizer; this.authorizer = authorizer;
} }
static class DatabaseList extends ResultList<Database> { public static class DatabaseList extends ResultList<Database> {
@SuppressWarnings("unused") // Empty constructor needed for deserialization @SuppressWarnings("unused") // Empty constructor needed for deserialization
DatabaseList() {} DatabaseList() {}
DatabaseList(List<Database> data, int limitParam, String beforeCursor, public DatabaseList(List<Database> data, String beforeCursor, String afterCursor, int total)
String afterCursor) throws GeneralSecurityException, UnsupportedEncodingException { throws GeneralSecurityException, UnsupportedEncodingException {
super(data, limitParam, beforeCursor, afterCursor); super(data, beforeCursor, afterCursor, total);
} }
} }
@ -158,29 +158,18 @@ public class DatabaseResource {
RestUtil.validateCursors(before, after); RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam); Fields fields = new Fields(FIELD_LIST, fieldsParam);
List<Database> databases; DatabaseList databases;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward // For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling, // scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results. // beforeCursor is not null. Remove the extra entry before returning results.
if (before != null) { // Reverse paging if (before != null) { // Reverse paging
databases = dao.listBefore(fields, serviceParam, limitParam + 1, before); // Ask for one extra entry databases = dao.listBefore(fields, serviceParam, limitParam, before); // Ask for one extra entry
if (databases.size() > limitParam) {
databases.remove(0);
beforeCursor = databases.get(0).getFullyQualifiedName();
}
afterCursor = databases.get(databases.size() - 1).getFullyQualifiedName();
} else { // Forward paging or first page } else { // Forward paging or first page
databases = dao.listAfter(fields, serviceParam, limitParam + 1, after); databases = dao.listAfter(fields, serviceParam, limitParam, after);
beforeCursor = after == null ? null : databases.get(0).getFullyQualifiedName();
if (databases.size() > limitParam) {
databases.remove(limitParam);
afterCursor = databases.get(limitParam - 1).getFullyQualifiedName();
}
} }
addHref(uriInfo, databases); addHref(uriInfo, databases.getData());
return new DatabaseList(databases, limitParam, beforeCursor, afterCursor); return databases;
} }
@GET @GET

View File

@ -109,9 +109,9 @@ public class TableResource {
@SuppressWarnings("unused") /* Required for tests */ @SuppressWarnings("unused") /* Required for tests */
public TableList() {} public TableList() {}
public TableList(List<Table> data, int limitParam, String beforeCursor, String afterCursor) public TableList(List<Table> data, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException { throws GeneralSecurityException, UnsupportedEncodingException {
super(data, limitParam, beforeCursor, afterCursor); super(data, beforeCursor, afterCursor, total);
} }
} }
@ -153,29 +153,14 @@ public class TableResource {
RestUtil.validateCursors(before, after); RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam); Fields fields = new Fields(FIELD_LIST, fieldsParam);
List<Table> tables; TableList tables;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
if (before != null) { // Reverse paging if (before != null) { // Reverse paging
tables = dao.listBefore(fields, databaseParam, limitParam + 1, before); // Ask for one extra entry tables = dao.listBefore(fields, databaseParam, limitParam, before);
if (tables.size() > limitParam) {
tables.remove(0);
beforeCursor = tables.get(0).getFullyQualifiedName();
}
afterCursor = tables.get(tables.size() - 1).getFullyQualifiedName();
} else { // Forward paging or first page } else { // Forward paging or first page
tables = dao.listAfter(fields, databaseParam, limitParam + 1, after); tables = dao.listAfter(fields, databaseParam, limitParam, after);
beforeCursor = after == null ? null : tables.get(0).getFullyQualifiedName();
if (tables.size() > limitParam) {
tables.remove(limitParam);
afterCursor = tables.get(limitParam - 1).getFullyQualifiedName();
}
} }
tables.forEach(t -> addHref(uriInfo, t)); tables.getData().forEach(t -> addHref(uriInfo, t));
return new TableList(tables, limitParam, beforeCursor, afterCursor); return tables;
} }
@GET @GET

View File

@ -97,13 +97,13 @@ public class TeamResource {
this.authorizer = authorizer; this.authorizer = authorizer;
} }
static class TeamList extends ResultList<Team> { public static class TeamList extends ResultList<Team> {
@SuppressWarnings("unused") /* Required for tests */ @SuppressWarnings("unused") /* Required for tests */
TeamList() {} TeamList() {}
TeamList(List<Team> teams, int limitParam, String beforeCursor, String afterCursor) public TeamList(List<Team> teams, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException { throws GeneralSecurityException, UnsupportedEncodingException {
super(teams, limitParam, beforeCursor, afterCursor); super(teams, beforeCursor, afterCursor, total);
} }
} }
@ -140,30 +140,15 @@ public class TeamResource {
@QueryParam("after") String after) throws IOException, GeneralSecurityException, ParseException { @QueryParam("after") String after) throws IOException, GeneralSecurityException, ParseException {
RestUtil.validateCursors(before, after); RestUtil.validateCursors(before, after);
EntityUtil.Fields fields = new EntityUtil.Fields(FIELD_LIST, fieldsParam); EntityUtil.Fields fields = new EntityUtil.Fields(FIELD_LIST, fieldsParam);
List<Team> teams;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward TeamList teams;
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
if (before != null) { // Reverse paging if (before != null) { // Reverse paging
teams = dao.listBefore(fields, limitParam + 1, before); // Ask for one extra entry teams = dao.listBefore(fields, limitParam, before); // Ask for one extra entry
if (teams.size() > limitParam) {
teams.remove(0);
beforeCursor = teams.get(0).getName();
}
afterCursor = teams.get(teams.size() - 1).getName();
} else { // Forward paging or first page } else { // Forward paging or first page
teams = dao.listAfter(fields, limitParam + 1, after); teams = dao.listAfter(fields, limitParam, after);
beforeCursor = after == null ? null : teams.get(0).getName();
if (teams.size() > limitParam) {
teams.remove(limitParam);
afterCursor = teams.get(limitParam - 1).getName();
}
} }
teams.forEach(team -> addHref(uriInfo, team)); teams.getData().forEach(team -> addHref(uriInfo, team));
LOG.info("Returning {} teams", teams.size()); return teams;
return new TeamList(teams, limitParam, beforeCursor, afterCursor);
} }
@GET @GET

View File

@ -100,13 +100,13 @@ public class UserResource {
this.authorizer = authorizer; this.authorizer = authorizer;
} }
static class UserList extends ResultList<User> { public static class UserList extends ResultList<User> {
@SuppressWarnings("unused") // Used for deserialization @SuppressWarnings("unused") // Used for deserialization
UserList() {} public UserList() {}
UserList(List<User> users, int limitParam, String beforeCursor, String afterCursor) public UserList(List<User> users, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException { throws GeneralSecurityException, UnsupportedEncodingException {
super(users, limitParam, beforeCursor, afterCursor); super(users, beforeCursor, afterCursor, total);
} }
} }
@ -144,29 +144,14 @@ public class UserResource {
RestUtil.validateCursors(before, after); RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam); Fields fields = new Fields(FIELD_LIST, fieldsParam);
List<User> users; UserList users;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
if (before != null) { // Reverse paging if (before != null) { // Reverse paging
users = dao.listBefore(fields, limitParam + 1, before); // Ask for one extra entry users = dao.listBefore(fields, limitParam, before);
if (users.size() > limitParam) {
users.remove(0);
beforeCursor = users.get(0).getName();
}
afterCursor = users.get(users.size() - 1).getName();
} else { // Forward paging or first page } else { // Forward paging or first page
users = dao.listAfter(fields, limitParam + 1, after); users = dao.listAfter(fields, limitParam, after);
beforeCursor = after == null ? null : users.get(0).getName();
if (users.size() > limitParam) {
users.remove(limitParam);
afterCursor = users.get(limitParam - 1).getName();
}
} }
Optional.ofNullable(users).orElse(Collections.emptyList()).forEach(u -> addHref(uriInfo, u)); Optional.ofNullable(users.getData()).orElse(Collections.emptyList()).forEach(u -> addHref(uriInfo, u));
return new UserList(users, limitParam, beforeCursor, afterCursor); return users;
} }
@GET @GET

View File

@ -28,7 +28,6 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponse;
import org.openmetadata.catalog.api.data.CreateTopic; import org.openmetadata.catalog.api.data.CreateTopic;
import org.openmetadata.catalog.entity.data.Dashboard; import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.entity.data.Topic; import org.openmetadata.catalog.entity.data.Topic;
import org.openmetadata.catalog.jdbi3.TopicRepository; import org.openmetadata.catalog.jdbi3.TopicRepository;
import org.openmetadata.catalog.resources.Collection; import org.openmetadata.catalog.resources.Collection;
@ -109,15 +108,15 @@ public class TopicResource {
this.authorizer = authorizer; this.authorizer = authorizer;
} }
static class TopicList extends ResultList<Topic> { public static class TopicList extends ResultList<Topic> {
@SuppressWarnings("unused") @SuppressWarnings("unused")
TopicList() { public TopicList() {
// Empty constructor needed for deserialization // Empty constructor needed for deserialization
} }
TopicList(List<Topic> data, int limitParam, String beforeCursor, public TopicList(List<Topic> data, String beforeCursor, String afterCursor, int total)
String afterCursor) throws GeneralSecurityException, UnsupportedEncodingException { throws GeneralSecurityException, UnsupportedEncodingException {
super(data, limitParam, beforeCursor, afterCursor); super(data, beforeCursor, afterCursor, total);
} }
} }
@ -159,29 +158,14 @@ public class TopicResource {
RestUtil.validateCursors(before, after); RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam); Fields fields = new Fields(FIELD_LIST, fieldsParam);
List<Topic> topics; TopicList topics;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
if (before != null) { // Reverse paging if (before != null) { // Reverse paging
topics = dao.listBefore(fields, serviceParam, limitParam + 1, before); // Ask for one extra entry topics = dao.listBefore(fields, serviceParam, limitParam, before); // Ask for one extra entry
if (topics.size() > limitParam) {
topics.remove(0);
beforeCursor = topics.get(0).getFullyQualifiedName();
}
afterCursor = topics.get(topics.size() - 1).getFullyQualifiedName();
} else { // Forward paging or first page } else { // Forward paging or first page
topics = dao.listAfter(fields, serviceParam, limitParam + 1, after); topics = dao.listAfter(fields, serviceParam, limitParam, after);
beforeCursor = after == null ? null : topics.get(0).getFullyQualifiedName();
if (topics.size() > limitParam) {
topics.remove(limitParam);
afterCursor = topics.get(limitParam - 1).getFullyQualifiedName();
}
} }
addHref(uriInfo, topics); addHref(uriInfo, topics.getData());
return new TopicList(topics, limitParam, beforeCursor, afterCursor); return topics;
} }
@GET @GET

View File

@ -19,6 +19,7 @@ package org.openmetadata.catalog.util;
import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import org.openmetadata.catalog.type.Paging;
import org.openmetadata.common.utils.CipherText; import org.openmetadata.common.utils.CipherText;
import javax.validation.constraints.NotNull; import javax.validation.constraints.NotNull;
@ -39,38 +40,6 @@ import java.util.List;
@JsonPropertyOrder({"data"}) @JsonPropertyOrder({"data"})
public class ResultList<T> { public class ResultList<T> {
public static class Paging {
@JsonProperty("before")
@NotNull
private String before;
@JsonProperty("after")
@NotNull
private String after;
@JsonProperty("before")
public String getBefore() {
return before;
}
@JsonProperty("before")
public Paging setBefore(String before) {
this.before = before;
return this;
}
@JsonProperty("after")
public String getAfter() {
return after;
}
@JsonProperty("after")
public Paging setAfter(String after) {
this.after = after;
return this;
}
}
@JsonProperty("data") @JsonProperty("data")
@NotNull @NotNull
private List<T> data; private List<T> data;
@ -139,12 +108,11 @@ public class ResultList<T> {
* -------- BACKWARD SCROLLING ENDS ------------- * -------- BACKWARD SCROLLING ENDS -------------
* *
*/ */
public ResultList(List<T> data, int limit, String beforeCursor, String afterCursor) throws GeneralSecurityException, public ResultList(List<T> data, String beforeCursor, String afterCursor, int total) throws GeneralSecurityException,
UnsupportedEncodingException { UnsupportedEncodingException {
this.data = data; this.data = data;
paging = new Paging(); paging = new Paging().withBefore(CipherText.instance().encrypt(beforeCursor))
paging.before = CipherText.instance().encrypt(beforeCursor); .withAfter(CipherText.instance().encrypt(afterCursor)).withTotal(total);
paging.after = CipherText.instance().encrypt(afterCursor);
} }
@JsonProperty("data") @JsonProperty("data")
@ -167,5 +135,4 @@ public class ResultList<T> {
this.paging = paging; this.paging = paging;
return this; return this;
} }
} }

View File

@ -34,6 +34,7 @@
}, },
"timeInterval": { "timeInterval": {
"type": "object", "type": "object",
"description": "Time interval in unixTimeMillis.",
"javaType": "org.openmetadata.catalog.type.TimeInterval", "javaType": "org.openmetadata.catalog.type.TimeInterval",
"properties": { "properties": {
"start": { "start": {

View File

@ -0,0 +1,23 @@
{
"$id": "https://open-metadata.org/schema/type/paging.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Paging",
"description": "Type used for cursor based pagination information in GET list responses.",
"type": "object",
"javaType": "org.openmetadata.catalog.type.Paging",
"properties": {
"before": {
"description": "Before cursor used for getting the previous page (see API pagination for details).",
"type": "string"
},
"after": {
"description": "After cursor used for getting the next page (see API pagination for details).",
"type": "string"
},
"total": {
"description": "Total number of entries available to page through.",
"type" : "integer"
}
},
"required": ["total"]
}

View File

@ -121,6 +121,8 @@ public final class TestUtils {
for (int i = 0; i < actual.getData().size(); i++) { for (int i = 0; i < actual.getData().size(); i++) {
assertEquals(allEntities.get(offset + i), actual.getData().get(i)); assertEquals(allEntities.get(offset + i), actual.getData().get(i));
} }
// Ensure total count returned in paging is correct
assertEquals(allEntities.size(), actual.getPaging().getTotal());
} }
public static void post(WebTarget target, Map<String, String> headers) throws HttpResponseException { public static void post(WebTarget target, Map<String, String> headers) throws HttpResponseException {