yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-03 02:55:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #800 - Remove boiler plate code for listAfter and listBefore methods in TableRepository

Browse Source
This commit is contained in:
sureshms 2021-10-16 10:02:09 -07:00
parent d15ffd8daf
commit fcf71a1d81
4 changed files with 134 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java Normal file
View File

@ -0,0 +1,33 @@
package org.openmetadata.catalog.jdbi3;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.ResultList;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.List;
/**
* Interface used for accessing the concrete entity DAOs such as table, dashboard etc.
* This gives a uniform access so that common boiler plate code can be reduced.
*/
public interface EntityRepository<T> {
/**
* DAO related operations
*/
List<String> listAfter(String fqnPrefix, int limitParam, String after);
List<String> listBefore(String fqnPrefix, int limitParam, String before);
int listCount(String fqnPrefix);
/**
* Entity related operations
*/
String getFullyQualifiedName(T entity);
T setFields(T entity, Fields fields) throws IOException, ParseException;
ResultList<T> getResultList(List<T> entities, String beforeCursor, String afterCursor,
int total) throws GeneralSecurityException, UnsupportedEncodingException;
}

78
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java
View File

@ -45,6 +45,7 @@ import org.openmetadata.catalog.util.EventUtils;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.RestUtil;
import org.openmetadata.catalog.util.RestUtil.PutResponse;
import org.openmetadata.catalog.util.ResultList;
import org.openmetadata.common.utils.CipherText;
import org.openmetadata.common.utils.CommonUtil;
import org.skife.jdbi.v2.sqlobject.Bind;
@ -58,6 +59,7 @@ import org.slf4j.LoggerFactory;
import javax.json.JsonPatch;
import javax.ws.rs.core.Response.Status;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.ArrayList;
@ -113,47 +115,49 @@ public abstract class TableRepository {
@CreateSqlObject
abstract TagDAO tagDAO();
EntityRepository<Table> entityRepository = new EntityRepository<Table>() {
@Override
public List<String> listAfter(String fqnPrefix, int limitParam, String after) {
return tableDAO().listAfter(fqnPrefix, limitParam, after);
}
@Override
public List<String> listBefore(String fqnPrefix, int limitParam, String after) {
return tableDAO().listBefore(fqnPrefix, limitParam, after);
}
@Override
public int listCount(String fqnPrefix) {
return tableDAO().listCount(fqnPrefix);
}
@Override
public String getFullyQualifiedName(Table entity) {
return entity.getFullyQualifiedName();
}
@Override
public Table setFields(Table entity, Fields fields) throws IOException, ParseException {
return TableRepository.this.setFields(entity, fields);
}
@Override
public ResultList<Table> getResultList(List<Table> entities, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
return new TableList(entities, beforeCursor, afterCursor, total);
}
};
@Transaction
public TableList listAfter(Fields fields, String databaseFQN, int limitParam, String after) throws IOException,
ParseException, GeneralSecurityException {
// forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = tableDAO().listAfter(databaseFQN, limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after));
List<Table> tables = new ArrayList<>();
for (String json : jsons) {
tables.add(setFields(JsonUtils.readValue(json, Table.class), fields));
}
int total = tableDAO().listCount(databaseFQN);
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : tables.get(0).getFullyQualifiedName();
if (tables.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
tables.remove(limitParam);
afterCursor = tables.get(limitParam - 1).getFullyQualifiedName();
}
return new TableList(tables, beforeCursor, afterCursor, total);
public ResultList<Table> listAfter(Fields fields, String databaseFQN, int limitParam, String after)
throws IOException, ParseException, GeneralSecurityException {
return EntityUtil.listAfter(entityRepository, Table.class, fields, databaseFQN, limitParam, after);
}
@Transaction
public TableList listBefore(Fields fields, String databaseFQN, int limitParam, String before) throws IOException,
ParseException, GeneralSecurityException {
// Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = tableDAO().listBefore(databaseFQN, limitParam + 1, CipherText.instance().decrypt(before));
List<Table> tables = new ArrayList<>();
for (String json : jsons) {
tables.add(setFields(JsonUtils.readValue(json, Table.class), fields));
}
int total = tableDAO().listCount(databaseFQN);
String beforeCursor = null, afterCursor;
if (tables.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
tables.remove(0);
beforeCursor = tables.get(0).getFullyQualifiedName();
}
afterCursor = tables.get(tables.size() - 1).getFullyQualifiedName();
return new TableList(tables, beforeCursor, afterCursor, total);
public ResultList<Table> listBefore(Fields fields, String databaseFQN, int limitParam, String before)
throws IOException, ParseException, GeneralSecurityException {
return EntityUtil.listBefore(entityRepository, Table.class, fields, databaseFQN, limitParam, before);
}
@Transaction

16
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java
View File

@ -130,31 +130,31 @@ public class TableResource {
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = TableList.class)))
})
public TableList list(@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Fields requested in the returned resource",
public ResultList<Table> list(@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields") String fieldsParam,
@Parameter(description = "Filter tables by database fully qualified name",
@Parameter(description = "Filter tables by database fully qualified name",
schema = @Schema(type = "string", example = "snowflakeWestCoast.financeDB"))
@QueryParam("database") String databaseParam,
@Parameter(description = "Limit the number tables returned. (1 to 1000000, default = 10) ",
@Parameter(description = "Limit the number tables returned. (1 to 1000000, default = 10) ",
schema = @Schema(type = "string", example = "snowflakeWestCoast.financeDB"))
@DefaultValue("10")
@Min(1)
@Max(1000000)
@QueryParam("limit") int limitParam,
@Parameter(description = "Returns list of tables before this cursor",
@Parameter(description = "Returns list of tables before this cursor",
schema = @Schema(type = "string"))
@QueryParam("before") String before,
@Parameter(description = "Returns list of tables after this cursor",
@Parameter(description = "Returns list of tables after this cursor",
schema = @Schema(type = "string"))
@QueryParam("after") String after)
throws IOException, ParseException, GeneralSecurityException {
RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam);
TableList tables;
ResultList<Table> tables;
if (before != null) { // Reverse paging
tables = dao.listBefore(fields, databaseParam, limitParam, before);
} else { // Forward paging or first page

57
catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java
View File

@ -38,6 +38,7 @@ import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO;
import org.openmetadata.catalog.jdbi3.DashboardRepository.DashboardDAO;
import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseDAO;
import org.openmetadata.catalog.jdbi3.EntityRepository;
import org.openmetadata.catalog.jdbi3.EntityRelationshipDAO;
import org.openmetadata.catalog.jdbi3.MetricsRepository.MetricsDAO;
import org.openmetadata.catalog.jdbi3.ModelRepository.ModelDAO;
@ -72,12 +73,15 @@ import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.TagLabel.LabelType;
import org.openmetadata.catalog.type.UsageDetails;
import org.openmetadata.catalog.type.UsageStats;
import org.openmetadata.common.utils.CipherText;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.UriInfo;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@ -164,7 +168,7 @@ public final class EntityUtil {
} else if (entity.equalsIgnoreCase(Entity.PIPELINE_SERVICE)) {
PipelineServiceResource.addHref(uriInfo, ref);
} else {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(ref.getType()));
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(ref.getType()));
}
}
@ -185,7 +189,7 @@ public final class EntityUtil {
if (ids.size() > 1) {
LOG.warn("Possible database issues - multiple owners {} found for entity {}", ids, id);
}
return ids.isEmpty() ? null : EntityUtil.populateOwner(userDAO, teamDAO, ids.get(0));
return ids.isEmpty() ? null : EntityUtil.populateOwner(userDAO, teamDAO, ids.get(0));
}
/**
@ -293,7 +297,7 @@ public final class EntityUtil {
} else if (entity.equalsIgnoreCase(Entity.PIPELINE)) {
Pipeline instance = EntityUtil.validate(id, pipelineDAO.findById(id), Pipeline.class);
return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName());
} else if (entity.equalsIgnoreCase(Entity.MODEL)) {
} else if (entity.equalsIgnoreCase(Entity.MODEL)) {
Model instance = EntityUtil.validate(id, modelDAO.findById(id), Model.class);
return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName());
}
@ -340,7 +344,7 @@ public final class EntityUtil {
} else if (entity.equalsIgnoreCase(Entity.TASK)) {
Task instance = EntityUtil.validate(fqn, taskDAO.findByFQN(fqn), Task.class);
return getEntityReference(instance);
} else if (entity.equalsIgnoreCase(Entity.PIPELINE)) {
} else if (entity.equalsIgnoreCase(Entity.PIPELINE)) {
Pipeline instance = EntityUtil.validate(fqn, pipelineDAO.findByFQN(fqn), Pipeline.class);
return getEntityReference(instance);
} else if (entity.equalsIgnoreCase(Entity.MODEL)) {
@ -431,7 +435,7 @@ public final class EntityUtil {
throws IOException {
String entityType = entityLink.getEntityType();
String fqn = entityLink.getEntityId();
if(entityType.equalsIgnoreCase(Entity.USER)) {
if (entityType.equalsIgnoreCase(Entity.USER)) {
return getEntityReference(EntityUtil.validate(fqn, userDAO.findByName(fqn), User.class));
} else if (entityType.equalsIgnoreCase(Entity.TEAM)) {
return getEntityReference(EntityUtil.validate(fqn, teamDAO.findByName(fqn), Team.class));
@ -650,4 +654,47 @@ public final class EntityUtil {
return fieldList.contains(field);
}
}
public static <T> ResultList<T> listAfter(EntityRepository<T> dao, Class<T> clz, Fields fields, String prefixFqn,
int limitParam, String after)
throws IOException, ParseException, GeneralSecurityException {
// forward scrolling, if after == null then first page is being asked being asked
List<String> jsons = dao.listAfter(prefixFqn, limitParam + 1, after == null ? "" :
CipherText.instance().decrypt(after));
List<T> entities = new ArrayList<>();
for (String json : jsons) {
entities.add(dao.setFields(JsonUtils.readValue(json, clz), fields));
}
int total = dao.listCount(prefixFqn);
String beforeCursor, afterCursor = null;
beforeCursor = after == null ? null : dao.getFullyQualifiedName(entities.get(0));
if (entities.size() > limitParam) { // If extra result exists, then next page exists - return after cursor
entities.remove(limitParam);
afterCursor = dao.getFullyQualifiedName(entities.get(limitParam - 1));
}
return dao.getResultList(entities, beforeCursor, afterCursor, total);
}
public static <T> ResultList<T> listBefore(EntityRepository<T> dao, Class<T> clz, Fields fields, String databaseFQN,
int limitParam, String before)
throws IOException, ParseException, GeneralSecurityException {
// Reverse scrolling - Get one extra result used for computing before cursor
List<String> jsons = dao.listBefore(databaseFQN, limitParam + 1, CipherText.instance().decrypt(before));
List<T> entities = new ArrayList<>();
for (String json : jsons) {
entities.add(dao.setFields(JsonUtils.readValue(json, clz), fields));
}
int total = dao.listCount(databaseFQN);
String beforeCursor = null, afterCursor;
if (entities.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor
entities.remove(0);
beforeCursor = dao.getFullyQualifiedName(entities.get(0));
}
afterCursor = dao.getFullyQualifiedName(entities.get(entities.size() - 1));
return dao.getResultList(entities, beforeCursor, afterCursor, total);
}
}