yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-06-27 04:22:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #379 - Add total number of entities to list APIs (#411)

Browse Source 
Co-authored-by: sureshms <suresh@getcollate.io>
This commit is contained in:
Suresh Srinivas 2021-09-06 13:49:48 -07:00 committed by GitHub
parent 27a02fa379
commit 076bac5f2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 299 additions and 270 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogHealthCheck.java
View File

@ -17,13 +17,12 @@
package org.openmetadata.catalog;
import com.codahale.metrics.health.HealthCheck;
import org.openmetadata.catalog.entity.teams.User;
import org.openmetadata.catalog.jdbi3.UserRepository;
import org.openmetadata.catalog.resources.teams.UserResource.UserList;
import org.openmetadata.catalog.util.EntityUtil;
import org.skife.jdbi.v2.DBI;
import java.io.IOException;
import java.util.List;
import static org.openmetadata.catalog.resources.teams.UserResource.FIELD_LIST;
@ -39,7 +38,7 @@ public class CatalogHealthCheck extends HealthCheck {
@Override
protected Result check() throws Exception {
try {
List<User> users = userRepository.listAfter(fields, 1, "");
UserList users = userRepository.listAfter(fields, 1, "");
return Result.healthy();
} catch (IOException e) {
return Result.unhealthy(e.getMessage());

39
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java
View File

@ -25,6 +25,7 @@ import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServic
import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
import org.openmetadata.catalog.resources.charts.ChartResource;
import org.openmetadata.catalog.resources.charts.ChartResource.ChartList;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityUtil;
@ -79,29 +80,45 @@ public abstract class ChartRepository {
@Transaction
public List<Chart> listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException,
public ChartList listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException,
GeneralSecurityException {
// forward scrolling, either because after != null or first page is being asked
List<String> jsons = chartDAO().listAfter(serviceName, limitParam, after == null ? "" :
// forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = chartDAO().listAfter(serviceName, limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after));
List<Chart> charts = new ArrayList<>();
for (String json : jsons) {
charts.add(setFields(JsonUtils.readValue(json, Chart.class), fields));
}
return charts;
int total = chartDAO().listCount(serviceName);
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : charts.get(0).getFullyQualifiedName();
if (charts.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
charts.remove(limitParam);
afterCursor = charts.get(limitParam - 1).getFullyQualifiedName();
}
return new ChartList(charts, beforeCursor, afterCursor, total);
}
@Transaction
public List<Chart> listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException,
public ChartList listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException,
GeneralSecurityException {
// Reverse scrolling
List<String> jsons = chartDAO().listBefore(serviceName, limitParam, CipherText.instance().decrypt(before));
// Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = chartDAO().listBefore(serviceName, limitParam + 1, CipherText.instance().decrypt(before));
List<Chart> charts = new ArrayList<>();
for (String json : jsons) {
charts.add(setFields(JsonUtils.readValue(json, Chart.class), fields));
}
return charts;
int total = chartDAO().listCount(serviceName);
String beforeCursor = null, afterCursor;
if (charts.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
charts.remove(0);
beforeCursor = charts.get(0).getFullyQualifiedName();
}
afterCursor = charts.get(charts.size() - 1).getFullyQualifiedName();
return new ChartList(charts, beforeCursor, afterCursor, total);
}
@Transaction
@ -140,7 +157,7 @@ public abstract class ChartRepository {
String fqn = getFQN(service, updatedChart);
Chart storedDB = JsonUtils.readValue(chartDAO().findByFQN(fqn), Chart.class);
if (storedDB == null) { // Chart does not exist. Create a new one
return new PutResponse<Chart>(Status.CREATED, createInternal(updatedChart, service, newOwner));
return new PutResponse<>(Status.CREATED, createInternal(updatedChart, service, newOwner));
}
// Update the existing chart
EntityUtil.populateOwner(userDAO(), teamDAO(), newOwner); // Validate new owner
@ -307,6 +324,10 @@ public abstract class ChartRepository {
@SqlQuery("SELECT json FROM chart_entity WHERE id = :id")
String findById(@Bind("id") String id);
@SqlQuery("SELECT count(*) FROM chart_entity WHERE " +
"(fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)")
int listCount(@Bind("fqnPrefix") String fqnPrefix);
@SqlQuery(
"SELECT json FROM (" +
"SELECT fullyQualifiedName, json FROM chart_entity WHERE " +

38
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java
View File

@ -28,6 +28,7 @@ import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO;
import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO;
import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceDAO;
import org.openmetadata.catalog.resources.dashboards.DashboardResource.DashboardList;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityUtil;
@ -87,29 +88,45 @@ public abstract class DashboardRepository {
@Transaction
public List<Dashboard> listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException,
public DashboardList listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException,
GeneralSecurityException {
// forward scrolling, either because after != null or first page is being asked
List<String> jsons = dashboardDAO().listAfter(serviceName, limitParam, after == null ? "" :
// forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = dashboardDAO().listAfter(serviceName, limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after));
List<Dashboard> dashboards = new ArrayList<>();
for (String json : jsons) {
dashboards.add(setFields(JsonUtils.readValue(json, Dashboard.class), fields));
}
return dashboards;
int total = dashboardDAO().listCount(serviceName);
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : dashboards.get(0).getFullyQualifiedName();
if (dashboards.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
dashboards.remove(limitParam);
afterCursor = dashboards.get(limitParam - 1).getFullyQualifiedName();
}
return new DashboardList(dashboards, beforeCursor, afterCursor, total);
}
@Transaction
public List<Dashboard> listBefore(Fields fields, String serviceName, int limitParam, String before)
public DashboardList listBefore(Fields fields, String serviceName, int limitParam, String before)
throws IOException, GeneralSecurityException {
// Reverse scrolling
List<String> jsons = dashboardDAO().listBefore(serviceName, limitParam, CipherText.instance().decrypt(before));
// Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = dashboardDAO().listBefore(serviceName, limitParam + 1, CipherText.instance().decrypt(before));
List<Dashboard> dashboards = new ArrayList<>();
for (String json : jsons) {
dashboards.add(setFields(JsonUtils.readValue(json, Dashboard.class), fields));
}
return dashboards;
int total = dashboardDAO().listCount(serviceName);
String beforeCursor = null, afterCursor;
if (dashboards.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
dashboards.remove(0);
beforeCursor = dashboards.get(0).getFullyQualifiedName();
}
afterCursor = dashboards.get(dashboards.size() - 1).getFullyQualifiedName();
return new DashboardList(dashboards, beforeCursor, afterCursor, total);
}
@Transaction
@ -377,6 +394,10 @@ public abstract class DashboardRepository {
@SqlQuery("SELECT json FROM dashboard_entity WHERE fullyQualifiedName = :name")
String findByFQN(@Bind("name") String name);
@SqlQuery("SELECT count(*) FROM dashboard_entity WHERE " +
"(fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)")
int listCount(@Bind("fqnPrefix") String fqnPrefix);
@SqlQuery(
"SELECT json FROM (" +
"SELECT fullyQualifiedName, json FROM dashboard_entity WHERE " +
@ -397,7 +418,6 @@ public abstract class DashboardRepository {
List<String> listAfter(@Bind("fqnPrefix") String fqnPrefix, @Bind("limit") int limit,
@Bind("after") String after);
@SqlUpdate("DELETE FROM dashboard_entity WHERE id = :id")
int delete(@Bind("id") String id);
}

37
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseRepository.java
View File

@ -28,6 +28,7 @@ import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO;
import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
import org.openmetadata.catalog.resources.databases.DatabaseResource;
import org.openmetadata.catalog.resources.databases.DatabaseResource.DatabaseList;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields;
@ -93,29 +94,45 @@ public abstract class DatabaseRepository {
abstract UsageDAO usageDAO();
@Transaction
public List<Database> listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException,
public DatabaseList listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException,
GeneralSecurityException {
// forward scrolling, either because after != null or first page is being asked
List<String> jsons = databaseDAO().listAfter(serviceName, limitParam, after == null ? "" :
// forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = databaseDAO().listAfter(serviceName, limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after));
List<Database> databases = new ArrayList<>();
for (String json : jsons) {
databases.add(setFields(JsonUtils.readValue(json, Database.class), fields));
}
return databases;
int total = databaseDAO().listCount(serviceName);
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : databases.get(0).getFullyQualifiedName();
if (databases.size() > limitParam) {
databases.remove(limitParam);
afterCursor = databases.get(limitParam - 1).getFullyQualifiedName();
}
return new DatabaseList(databases, beforeCursor, afterCursor, total);
}
@Transaction
public List<Database> listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException,
public DatabaseList listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException,
GeneralSecurityException {
// Reverse scrolling
List<String> jsons = databaseDAO().listBefore(serviceName, limitParam, CipherText.instance().decrypt(before));
// Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = databaseDAO().listBefore(serviceName, limitParam + 1, CipherText.instance().decrypt(before));
List<Database> databases = new ArrayList<>();
for (String json : jsons) {
databases.add(setFields(JsonUtils.readValue(json, Database.class), fields));
}
return databases;
int total = databaseDAO().listCount(serviceName);
String beforeCursor = null, afterCursor;
if (databases.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
databases.remove(0);
beforeCursor = databases.get(0).getFullyQualifiedName();
}
afterCursor = databases.get(databases.size() - 1).getFullyQualifiedName();
return new DatabaseList(databases, beforeCursor, afterCursor, total);
}
@Transaction
@ -310,6 +327,10 @@ public abstract class DatabaseRepository {
@SqlQuery("SELECT json FROM database_entity WHERE id = :id")
String findById(@Bind("id") String id);
@SqlQuery("SELECT count(*) FROM database_entity WHERE " +
"(fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)")
int listCount(@Bind("fqnPrefix") String fqnPrefix);
@SqlQuery(
"SELECT json FROM (" +
"SELECT fullyQualifiedName, json FROM database_entity WHERE " +

45
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java
View File

@ -17,10 +17,6 @@
package org.openmetadata.catalog.jdbi3;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.openmetadata.catalog.type.ColumnJoin;
import org.openmetadata.catalog.type.JoinedWith;
import org.openmetadata.catalog.type.TableData;
import org.openmetadata.catalog.type.TableJoins;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Database;
import org.openmetadata.catalog.entity.data.Table;
@ -32,9 +28,14 @@ import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO;
import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
import org.openmetadata.catalog.resources.databases.TableResource;
import org.openmetadata.catalog.resources.databases.TableResource.TableList;
import org.openmetadata.catalog.type.Column;
import org.openmetadata.catalog.type.ColumnJoin;
import org.openmetadata.catalog.type.DailyCount;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.JoinedWith;
import org.openmetadata.catalog.type.TableData;
import org.openmetadata.catalog.type.TableJoins;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields;
@ -110,30 +111,46 @@ public abstract class TableRepository {
abstract TagDAO tagDAO();
@Transaction
public List<Table> listAfter(Fields fields, String databaseFQN, int limitParam, String after) throws IOException,
public TableList listAfter(Fields fields, String databaseFQN, int limitParam, String after) throws IOException,
ParseException, GeneralSecurityException {
// Forward scrolling, either because after != null or first page is being asked
List<String> jsons = tableDAO().listAfter(databaseFQN, limitParam, after == null ? "" :
// forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = tableDAO().listAfter(databaseFQN, limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after));
List<Table> tables = new ArrayList<>();
for (String json : jsons) {
tables.add(setFields(JsonUtils.readValue(json, Table.class), fields));
}
return tables;
int total = tableDAO().listCount(databaseFQN);
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : tables.get(0).getFullyQualifiedName();
if (tables.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
tables.remove(limitParam);
afterCursor = tables.get(limitParam - 1).getFullyQualifiedName();
}
return new TableList(tables, beforeCursor, afterCursor, total);
}
@Transaction
public List<Table> listBefore(Fields fields, String databaseFQN, int limitParam, String before) throws IOException,
public TableList listBefore(Fields fields, String databaseFQN, int limitParam, String before) throws IOException,
ParseException, GeneralSecurityException {
// Reverse scrolling
List<String> jsons = tableDAO().listBefore(databaseFQN, limitParam, CipherText.instance().decrypt(before));
// Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = tableDAO().listBefore(databaseFQN, limitParam + 1, CipherText.instance().decrypt(before));
List<Table> tables = new ArrayList<>();
for (String json : jsons) {
tables.add(setFields(JsonUtils.readValue(json, Table.class), fields));
}
return tables;
int total = tableDAO().listCount(databaseFQN);
String beforeCursor = null, afterCursor;
if (tables.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
tables.remove(0);
beforeCursor = tables.get(0).getFullyQualifiedName();
}
afterCursor = tables.get(tables.size() - 1).getFullyQualifiedName();
return new TableList(tables, beforeCursor, afterCursor, total);
}
@Transaction
@ -647,6 +664,10 @@ public abstract class TableRepository {
@SqlQuery("SELECT json FROM table_entity WHERE fullyQualifiedName = :tableFQN")
String findByFQN(@Bind("tableFQN") String tableFQN);
@SqlQuery("SELECT count(*) FROM table_entity WHERE " +
"(fullyQualifiedName LIKE CONCAT(:databaseFQN, '.%') OR :databaseFQN IS NULL)")
int listCount(@Bind("databaseFQN") String databseFQN);
@SqlQuery(
"SELECT json FROM (" +
"SELECT fullyQualifiedName, json FROM table_entity WHERE " +

34
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java
View File

@ -29,6 +29,7 @@ import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
import org.openmetadata.catalog.jdbi3.TopicRepository.TopicDAO;
import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO;
import org.openmetadata.catalog.resources.teams.TeamResource;
import org.openmetadata.catalog.resources.teams.TeamResource.TeamList;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields;
@ -127,30 +128,47 @@ public abstract class TeamRepository {
}
@Transaction
public List<Team> listAfter(Fields fields, int limitParam, String after) throws IOException,
public TeamList listAfter(Fields fields, int limitParam, String after) throws IOException,
ParseException, GeneralSecurityException {
// Forward scrolling, either because after != null or first page is being asked
List<String> jsons = teamDAO().listAfter(limitParam, after == null ? "" :
List<String> jsons = teamDAO().listAfter(limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after));
List<Team> teams = new ArrayList<>();
for (String json : jsons) {
teams.add(setFields(JsonUtils.readValue(json, Team.class), fields));
}
return teams;
int total = teamDAO().listCount();
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : teams.get(0).getName();
if (teams.size() > limitParam) {
teams.remove(limitParam);
afterCursor = teams.get(limitParam - 1).getName();
}
return new TeamList(teams, beforeCursor, afterCursor, total);
}
@Transaction
public List<Team> listBefore(Fields fields, int limitParam, String before) throws IOException,
public TeamList listBefore(Fields fields, int limitParam, String before) throws IOException,
ParseException, GeneralSecurityException {
// Reverse scrolling
List<String> jsons = teamDAO().listBefore(limitParam, CipherText.instance().decrypt(before));
List<String> jsons = teamDAO().listBefore(limitParam + 1, CipherText.instance().decrypt(before));
List<Team> teams = new ArrayList<>();
for (String json : jsons) {
teams.add(setFields(JsonUtils.readValue(json, Team.class), fields));
}
return teams;
int total = teamDAO().listCount();
String beforeCursor = null, afterCursor;
if (teams.size() > limitParam) {
teams.remove(0);
beforeCursor = teams.get(0).getName();
}
afterCursor = teams.get(teams.size() - 1).getName();
return new TeamList(teams, beforeCursor, afterCursor, total);
}
@Transaction
@ -257,6 +275,9 @@ public abstract class TeamRepository {
@SqlQuery("SELECT json FROM team_entity where name = :name")
String findByName(@Bind("name") String name);
@SqlQuery("SELECT count(*) FROM team_entity")
int listCount();
@SqlQuery(
"SELECT json FROM (" +
"SELECT name, json FROM team_entity WHERE " +
@ -272,7 +293,6 @@ public abstract class TeamRepository {
"LIMIT :limit")
List<String> listAfter(@Bind("limit") int limit, @Bind("after") String after);
@SqlUpdate("DELETE FROM team_entity WHERE id = :teamId")
int delete(@Bind("teamId") String teamId);

42
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java
View File

@ -17,16 +17,15 @@
package org.openmetadata.catalog.jdbi3;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.entity.data.Topic;
import org.openmetadata.catalog.entity.services.MessagingService;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceDAO;
import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO;
import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
import org.openmetadata.catalog.resources.topics.TopicResource;
import org.openmetadata.catalog.resources.topics.TopicResource.TopicList;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityUtil;
@ -79,33 +78,46 @@ public abstract class TopicRepository {
@CreateSqlObject
abstract TagRepository.TagDAO tagDAO();
@CreateSqlObject
abstract UsageDAO usageDAO();
@Transaction
public List<Topic> listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException,
public TopicList listAfter(Fields fields, String serviceName, int limitParam, String after) throws IOException,
GeneralSecurityException {
// forward scrolling, either because after != null or first page is being asked
List<String> jsons = topicDAO().listAfter(serviceName, limitParam, after == null ? "" :
// forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = topicDAO().listAfter(serviceName, limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after));
List<Topic> topics = new ArrayList<>();
for (String json : jsons) {
topics.add(setFields(JsonUtils.readValue(json, Topic.class), fields));
}
return topics;
int total = topicDAO().listCount(serviceName);
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : topics.get(0).getFullyQualifiedName();
if (topics.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
topics.remove(limitParam);
afterCursor = topics.get(limitParam - 1).getFullyQualifiedName();
}
return new TopicList(topics, beforeCursor, afterCursor, total);
}
@Transaction
public List<Topic> listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException,
public TopicList listBefore(Fields fields, String serviceName, int limitParam, String before) throws IOException,
GeneralSecurityException {
// Reverse scrolling
List<String> jsons = topicDAO().listBefore(serviceName, limitParam, CipherText.instance().decrypt(before));
// Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = topicDAO().listBefore(serviceName, limitParam + 1, CipherText.instance().decrypt(before));
List<Topic> topics = new ArrayList<>();
for (String json : jsons) {
topics.add(setFields(JsonUtils.readValue(json, Topic.class), fields));
}
return topics;
int total = topicDAO().listCount(serviceName);
String beforeCursor = null, afterCursor;
if (topics.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
topics.remove(0);
beforeCursor = topics.get(0).getFullyQualifiedName();
}
afterCursor = topics.get(topics.size() - 1).getFullyQualifiedName();
return new TopicList(topics, beforeCursor, afterCursor, total);
}
@Transaction
@ -309,6 +321,10 @@ public abstract class TopicRepository {
@SqlQuery("SELECT json FROM topic_entity WHERE id = :id")
String findById(@Bind("id") String id);
@SqlQuery("SELECT count(*) FROM topic_entity WHERE " +
"(fullyQualifiedName LIKE CONCAT(:fqnPrefix, '.%') OR :fqnPrefix IS NULL)") // Filter by service name
int listCount(@Bind("fqnPrefix") String fqnPrefix);
@SqlQuery(
"SELECT json FROM (" +
"SELECT fullyQualifiedName, json FROM topic_entity WHERE " +

36
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java
View File

@ -28,6 +28,7 @@ import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.TopicRepository.TopicDAO;
import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO;
import org.openmetadata.catalog.resources.teams.UserResource;
import org.openmetadata.catalog.resources.teams.UserResource.UserList;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields;
@ -101,30 +102,46 @@ public abstract class UserRepository {
abstract ChartDAO chartDAO();
@Transaction
public List<User> listAfter(Fields fields, int limitParam, String after) throws IOException,
public UserList listAfter(Fields fields, int limitParam, String after) throws IOException,
ParseException, GeneralSecurityException {
// Forward scrolling, either because after != null or first page is being asked
List<String> jsons = userDAO().listAfter(limitParam, after == null ? "" :
// forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = userDAO().listAfter(limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after));
List<User> users = new ArrayList<>();
for (String json : jsons) {
users.add(setFields(JsonUtils.readValue(json, User.class), fields));
}
return users;
int total = userDAO().listCount();
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : users.get(0).getName();
if (users.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
users.remove(limitParam);
afterCursor = users.get(limitParam - 1).getName();
}
return new UserList(users, beforeCursor, afterCursor, total);
}
@Transaction
public List<User> listBefore(Fields fields, int limitParam, String before) throws IOException,
public UserList listBefore(Fields fields, int limitParam, String before) throws IOException,
ParseException, GeneralSecurityException {
// Reverse scrolling
List<String> jsons = userDAO().listBefore(limitParam, CipherText.instance().decrypt(before));
// Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = userDAO().listBefore(limitParam + 1, CipherText.instance().decrypt(before));
List<User> users = new ArrayList<>();
for (String json : jsons) {
users.add(setFields(JsonUtils.readValue(json, User.class), fields));
}
return users;
int total = userDAO().listCount();
String beforeCursor = null, afterCursor;
if (users.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
users.remove(0);
beforeCursor = users.get(0).getName();
}
afterCursor = users.get(users.size() - 1).getName();
return new UserList(users, beforeCursor, afterCursor, total);
}
@Transaction
@ -309,6 +326,9 @@ public abstract class UserRepository {
@SqlQuery("SELECT json FROM user_entity")
List<String> list();
@SqlQuery("SELECT count(*) FROM user_entity")
int listCount();
@SqlQuery(
"SELECT json FROM (" +
"SELECT name, json FROM user_entity WHERE " +

33
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java
View File

@ -109,15 +109,15 @@ public class ChartResource {
this.authorizer = authorizer;
}
static class ChartList extends ResultList<Chart> {
public static class ChartList extends ResultList<Chart> {
@SuppressWarnings("unused")
ChartList() {
// Empty constructor needed for deserialization
}
ChartList(List<Chart> data, int limitParam, String beforeCursor,
String afterCursor) throws GeneralSecurityException, UnsupportedEncodingException {
super(data, limitParam, beforeCursor, afterCursor);
public ChartList(List<Chart> data, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(data, beforeCursor, afterCursor, total);
}
}
@ -159,29 +159,14 @@ public class ChartResource {
RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam);
List<Chart> charts;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
ChartList charts;
if (before != null) { // Reverse paging
charts = dao.listBefore(fields, serviceParam, limitParam + 1, before); // Ask for one extra entry
if (charts.size() > limitParam) {
charts.remove(0);
beforeCursor = charts.get(0).getFullyQualifiedName();
}
afterCursor = charts.get(charts.size() - 1).getFullyQualifiedName();
charts = dao.listBefore(fields, serviceParam, limitParam, before); // Ask for one extra entry
} else { // Forward paging or first page
charts = dao.listAfter(fields, serviceParam, limitParam + 1, after);
beforeCursor = after == null ? null : charts.get(0).getFullyQualifiedName();
if (charts.size() > limitParam) {
charts.remove(limitParam);
afterCursor = charts.get(limitParam - 1).getFullyQualifiedName();
charts = dao.listAfter(fields, serviceParam, limitParam, after);
}
}
addHref(uriInfo, charts);
return new ChartList(charts, limitParam, beforeCursor, afterCursor);
addHref(uriInfo, charts.getData());
return charts;
}
@GET

33
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java
View File

@ -112,15 +112,15 @@ public class DashboardResource {
this.authorizer = authorizer;
}
static class DashboardList extends ResultList<Dashboard> {
public static class DashboardList extends ResultList<Dashboard> {
@SuppressWarnings("unused")
DashboardList() {
// Empty constructor needed for deserialization
}
DashboardList(List<Dashboard> data, int limitParam, String beforeCursor,
String afterCursor) throws GeneralSecurityException, UnsupportedEncodingException {
super(data, limitParam, beforeCursor, afterCursor);
public DashboardList(List<Dashboard> data, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(data, beforeCursor, afterCursor, total);
}
}
@ -163,29 +163,14 @@ public class DashboardResource {
RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam);
List<Dashboard> dashboards;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
DashboardList dashboards;
if (before != null) { // Reverse paging
dashboards = dao.listBefore(fields, serviceParam, limitParam + 1, before); // Ask for one extra entry
if (dashboards.size() > limitParam) {
dashboards.remove(0);
beforeCursor = dashboards.get(0).getFullyQualifiedName();
}
afterCursor = dashboards.get(dashboards.size() - 1).getFullyQualifiedName();
dashboards = dao.listBefore(fields, serviceParam, limitParam, before); // Ask for one extra entry
} else { // Forward paging or first page
dashboards = dao.listAfter(fields, serviceParam, limitParam + 1, after);
beforeCursor = after == null ? null : dashboards.get(0).getFullyQualifiedName();
if (dashboards.size() > limitParam) {
dashboards.remove(limitParam);
afterCursor = dashboards.get(limitParam - 1).getFullyQualifiedName();
dashboards = dao.listAfter(fields, serviceParam, limitParam, after);
}
}
addHref(uriInfo, dashboards);
return new DashboardList(dashboards, limitParam, beforeCursor, afterCursor);
addHref(uriInfo, dashboards.getData());
return dashboards;
}
@GET

29
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseResource.java
View File

@ -111,12 +111,12 @@ public class DatabaseResource {
this.authorizer = authorizer;
}
static class DatabaseList extends ResultList<Database> {
public static class DatabaseList extends ResultList<Database> {
@SuppressWarnings("unused") // Empty constructor needed for deserialization
DatabaseList() {}
DatabaseList(List<Database> data, int limitParam, String beforeCursor,
String afterCursor) throws GeneralSecurityException, UnsupportedEncodingException {
super(data, limitParam, beforeCursor, afterCursor);
public DatabaseList(List<Database> data, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(data, beforeCursor, afterCursor, total);
}
}
@ -158,29 +158,18 @@ public class DatabaseResource {
RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam);
List<Database> databases;
String beforeCursor = null, afterCursor = null;
DatabaseList databases;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
if (before != null) { // Reverse paging
databases = dao.listBefore(fields, serviceParam, limitParam + 1, before); // Ask for one extra entry
if (databases.size() > limitParam) {
databases.remove(0);
beforeCursor = databases.get(0).getFullyQualifiedName();
}
afterCursor = databases.get(databases.size() - 1).getFullyQualifiedName();
databases = dao.listBefore(fields, serviceParam, limitParam, before); // Ask for one extra entry
} else { // Forward paging or first page
databases = dao.listAfter(fields, serviceParam, limitParam + 1, after);
beforeCursor = after == null ? null : databases.get(0).getFullyQualifiedName();
if (databases.size() > limitParam) {
databases.remove(limitParam);
afterCursor = databases.get(limitParam - 1).getFullyQualifiedName();
databases = dao.listAfter(fields, serviceParam, limitParam, after);
}
}
addHref(uriInfo, databases);
return new DatabaseList(databases, limitParam, beforeCursor, afterCursor);
addHref(uriInfo, databases.getData());
return databases;
}
@GET

29
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java
View File

@ -109,9 +109,9 @@ public class TableResource {
@SuppressWarnings("unused") /* Required for tests */
public TableList() {}
public TableList(List<Table> data, int limitParam, String beforeCursor, String afterCursor)
public TableList(List<Table> data, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(data, limitParam, beforeCursor, afterCursor);
super(data, beforeCursor, afterCursor, total);
}
}
@ -153,29 +153,14 @@ public class TableResource {
RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam);
List<Table> tables;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
TableList tables;
if (before != null) { // Reverse paging
tables = dao.listBefore(fields, databaseParam, limitParam + 1, before); // Ask for one extra entry
if (tables.size() > limitParam) {
tables.remove(0);
beforeCursor = tables.get(0).getFullyQualifiedName();
}
afterCursor = tables.get(tables.size() - 1).getFullyQualifiedName();
tables = dao.listBefore(fields, databaseParam, limitParam, before);
} else { // Forward paging or first page
tables = dao.listAfter(fields, databaseParam, limitParam + 1, after);
beforeCursor = after == null ? null : tables.get(0).getFullyQualifiedName();
if (tables.size() > limitParam) {
tables.remove(limitParam);
afterCursor = tables.get(limitParam - 1).getFullyQualifiedName();
tables = dao.listAfter(fields, databaseParam, limitParam, after);
}
}
tables.forEach(t -> addHref(uriInfo, t));
return new TableList(tables, limitParam, beforeCursor, afterCursor);
tables.getData().forEach(t -> addHref(uriInfo, t));
return tables;
}
@GET

31
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/TeamResource.java
View File

@ -97,13 +97,13 @@ public class TeamResource {
this.authorizer = authorizer;
}
static class TeamList extends ResultList<Team> {
public static class TeamList extends ResultList<Team> {
@SuppressWarnings("unused") /* Required for tests */
TeamList() {}
TeamList(List<Team> teams, int limitParam, String beforeCursor, String afterCursor)
public TeamList(List<Team> teams, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(teams, limitParam, beforeCursor, afterCursor);
super(teams, beforeCursor, afterCursor, total);
}
}
@ -140,30 +140,15 @@ public class TeamResource {
@QueryParam("after") String after) throws IOException, GeneralSecurityException, ParseException {
RestUtil.validateCursors(before, after);
EntityUtil.Fields fields = new EntityUtil.Fields(FIELD_LIST, fieldsParam);
List<Team> teams;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
TeamList teams;
if (before != null) { // Reverse paging
teams = dao.listBefore(fields, limitParam + 1, before); // Ask for one extra entry
if (teams.size() > limitParam) {
teams.remove(0);
beforeCursor = teams.get(0).getName();
}
afterCursor = teams.get(teams.size() - 1).getName();
teams = dao.listBefore(fields, limitParam, before); // Ask for one extra entry
} else { // Forward paging or first page
teams = dao.listAfter(fields, limitParam + 1, after);
beforeCursor = after == null ? null : teams.get(0).getName();
if (teams.size() > limitParam) {
teams.remove(limitParam);
afterCursor = teams.get(limitParam - 1).getName();
teams = dao.listAfter(fields, limitParam, after);
}
}
teams.forEach(team -> addHref(uriInfo, team));
LOG.info("Returning {} teams", teams.size());
return new TeamList(teams, limitParam, beforeCursor, afterCursor);
teams.getData().forEach(team -> addHref(uriInfo, team));
return teams;
}
@GET

33
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/UserResource.java
View File

@ -100,13 +100,13 @@ public class UserResource {
this.authorizer = authorizer;
}
static class UserList extends ResultList<User> {
public static class UserList extends ResultList<User> {
@SuppressWarnings("unused") // Used for deserialization
UserList() {}
public UserList() {}
UserList(List<User> users, int limitParam, String beforeCursor, String afterCursor)
public UserList(List<User> users, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(users, limitParam, beforeCursor, afterCursor);
super(users, beforeCursor, afterCursor, total);
}
}
@ -144,29 +144,14 @@ public class UserResource {
RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam);
List<User> users;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
UserList users;
if (before != null) { // Reverse paging
users = dao.listBefore(fields, limitParam + 1, before); // Ask for one extra entry
if (users.size() > limitParam) {
users.remove(0);
beforeCursor = users.get(0).getName();
}
afterCursor = users.get(users.size() - 1).getName();
users = dao.listBefore(fields, limitParam, before);
} else { // Forward paging or first page
users = dao.listAfter(fields, limitParam + 1, after);
beforeCursor = after == null ? null : users.get(0).getName();
if (users.size() > limitParam) {
users.remove(limitParam);
afterCursor = users.get(limitParam - 1).getName();
users = dao.listAfter(fields, limitParam, after);
}
}
Optional.ofNullable(users).orElse(Collections.emptyList()).forEach(u -> addHref(uriInfo, u));
return new UserList(users, limitParam, beforeCursor, afterCursor);
Optional.ofNullable(users.getData()).orElse(Collections.emptyList()).forEach(u -> addHref(uriInfo, u));
return users;
}
@GET

36
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java
View File

@ -28,7 +28,6 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import org.openmetadata.catalog.api.data.CreateTopic;
import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.entity.data.Topic;
import org.openmetadata.catalog.jdbi3.TopicRepository;
import org.openmetadata.catalog.resources.Collection;
@ -109,15 +108,15 @@ public class TopicResource {
this.authorizer = authorizer;
}
static class TopicList extends ResultList<Topic> {
public static class TopicList extends ResultList<Topic> {
@SuppressWarnings("unused")
TopicList() {
public TopicList() {
// Empty constructor needed for deserialization
}
TopicList(List<Topic> data, int limitParam, String beforeCursor,
String afterCursor) throws GeneralSecurityException, UnsupportedEncodingException {
super(data, limitParam, beforeCursor, afterCursor);
public TopicList(List<Topic> data, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(data, beforeCursor, afterCursor, total);
}
}
@ -159,29 +158,14 @@ public class TopicResource {
RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam);
List<Topic> topics;
String beforeCursor = null, afterCursor = null;
// For calculating cursors, ask for one extra entry beyond limit. If the extra entry exists, then in forward
// scrolling afterCursor is not null. Similarly, if the extra entry exists, then in reverse scrolling,
// beforeCursor is not null. Remove the extra entry before returning results.
TopicList topics;
if (before != null) { // Reverse paging
topics = dao.listBefore(fields, serviceParam, limitParam + 1, before); // Ask for one extra entry
if (topics.size() > limitParam) {
topics.remove(0);
beforeCursor = topics.get(0).getFullyQualifiedName();
}
afterCursor = topics.get(topics.size() - 1).getFullyQualifiedName();
topics = dao.listBefore(fields, serviceParam, limitParam, before); // Ask for one extra entry
} else { // Forward paging or first page
topics = dao.listAfter(fields, serviceParam, limitParam + 1, after);
beforeCursor = after == null ? null : topics.get(0).getFullyQualifiedName();
if (topics.size() > limitParam) {
topics.remove(limitParam);
afterCursor = topics.get(limitParam - 1).getFullyQualifiedName();
topics = dao.listAfter(fields, serviceParam, limitParam, after);
}
}
addHref(uriInfo, topics);
return new TopicList(topics, limitParam, beforeCursor, afterCursor);
addHref(uriInfo, topics.getData());
return topics;
}
@GET

41
catalog-rest-service/src/main/java/org/openmetadata/catalog/util/ResultList.java
View File

@ -19,6 +19,7 @@ package org.openmetadata.catalog.util;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import org.openmetadata.catalog.type.Paging;
import org.openmetadata.common.utils.CipherText;
import javax.validation.constraints.NotNull;
@ -39,38 +40,6 @@ import java.util.List;
@JsonPropertyOrder({"data"})
public class ResultList<T> {
public static class Paging {
@JsonProperty("before")
@NotNull
private String before;
@JsonProperty("after")
@NotNull
private String after;
@JsonProperty("before")
public String getBefore() {
return before;
}
@JsonProperty("before")
public Paging setBefore(String before) {
this.before = before;
return this;
}
@JsonProperty("after")
public String getAfter() {
return after;
}
@JsonProperty("after")
public Paging setAfter(String after) {
this.after = after;
return this;
}
}
@JsonProperty("data")
@NotNull
private List<T> data;
@ -139,12 +108,11 @@ public class ResultList<T> {
* -------- BACKWARD SCROLLING ENDS -------------
*
*/
public ResultList(List<T> data, int limit, String beforeCursor, String afterCursor) throws GeneralSecurityException,
public ResultList(List<T> data, String beforeCursor, String afterCursor, int total) throws GeneralSecurityException,
UnsupportedEncodingException {
this.data = data;
paging = new Paging();
paging.before = CipherText.instance().encrypt(beforeCursor);
paging.after = CipherText.instance().encrypt(afterCursor);
paging = new Paging().withBefore(CipherText.instance().encrypt(beforeCursor))
.withAfter(CipherText.instance().encrypt(afterCursor)).withTotal(total);
}
@JsonProperty("data")
@ -167,5 +135,4 @@ public class ResultList<T> {
this.paging = paging;
return this;
}
}

1
catalog-rest-service/src/main/resources/json/schema/type/basic.json
View File

@ -34,6 +34,7 @@
},
"timeInterval": {
"type": "object",
"description": "Time interval in unixTimeMillis.",
"javaType": "org.openmetadata.catalog.type.TimeInterval",
"properties": {
"start": {

23
catalog-rest-service/src/main/resources/json/schema/type/paging.json Normal file
View File

@ -0,0 +1,23 @@
{
"$id": "https://open-metadata.org/schema/type/paging.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Paging",
"description": "Type used for cursor based pagination information in GET list responses.",
"type": "object",
"javaType": "org.openmetadata.catalog.type.Paging",
"properties": {
"before": {
"description": "Before cursor used for getting the previous page (see API pagination for details).",
"type": "string"
},
"after": {
"description": "After cursor used for getting the next page (see API pagination for details).",
"type": "string"
},
"total": {
"description": "Total number of entries available to page through.",
"type" : "integer"
}
},
"required": ["total"]
}

2
catalog-rest-service/src/test/java/org/openmetadata/catalog/util/TestUtils.java
View File

@ -121,6 +121,8 @@ public final class TestUtils {
for (int i = 0; i < actual.getData().size(); i++) {
assertEquals(allEntities.get(offset + i), actual.getData().get(i));
}
// Ensure total count returned in paging is correct
assertEquals(allEntities.size(), actual.getPaging().getTotal());
}
public static void post(WebTarget target, Map<String, String> headers) throws HttpResponseException {