From fcf71a1d810b3b32dfcfb10fed6bbc28069456da Mon Sep 17 00:00:00 2001 From: sureshms Date: Sat, 16 Oct 2021 10:02:09 -0700 Subject: [PATCH] Fix #800 - Remove boiler plate code for listAfter and listBefore methods in TableRepository --- .../catalog/jdbi3/EntityRepository.java | 33 ++++++++ .../catalog/jdbi3/TableRepository.java | 78 ++++++++++--------- .../resources/databases/TableResource.java | 16 ++-- .../openmetadata/catalog/util/EntityUtil.java | 57 ++++++++++++-- 4 files changed, 134 insertions(+), 50 deletions(-) create mode 100644 catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java new file mode 100644 index 00000000000..65727e82e9a --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java @@ -0,0 +1,33 @@ +package org.openmetadata.catalog.jdbi3; + +import org.openmetadata.catalog.util.EntityUtil.Fields; +import org.openmetadata.catalog.util.ResultList; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.security.GeneralSecurityException; +import java.text.ParseException; +import java.util.List; + +/** + * Interface used for accessing the concrete entity DAOs such as table, dashboard etc. + * This gives a uniform access so that common boiler plate code can be reduced. + */ +public interface EntityRepository { + /** + * DAO related operations + */ + List listAfter(String fqnPrefix, int limitParam, String after); + List listBefore(String fqnPrefix, int limitParam, String before); + int listCount(String fqnPrefix); + + + /** + * Entity related operations + */ + String getFullyQualifiedName(T entity); + T setFields(T entity, Fields fields) throws IOException, ParseException; + + ResultList getResultList(List entities, String beforeCursor, String afterCursor, + int total) throws GeneralSecurityException, UnsupportedEncodingException; +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java index b57d3690de5..d96a79760b2 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java @@ -45,6 +45,7 @@ import org.openmetadata.catalog.util.EventUtils; import org.openmetadata.catalog.util.JsonUtils; import org.openmetadata.catalog.util.RestUtil; import org.openmetadata.catalog.util.RestUtil.PutResponse; +import org.openmetadata.catalog.util.ResultList; import org.openmetadata.common.utils.CipherText; import org.openmetadata.common.utils.CommonUtil; import org.skife.jdbi.v2.sqlobject.Bind; @@ -58,6 +59,7 @@ import org.slf4j.LoggerFactory; import javax.json.JsonPatch; import javax.ws.rs.core.Response.Status; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.security.GeneralSecurityException; import java.text.ParseException; import java.util.ArrayList; @@ -113,47 +115,49 @@ public abstract class TableRepository { @CreateSqlObject abstract TagDAO tagDAO(); + EntityRepository entityRepository = new EntityRepository
() { + @Override + public List listAfter(String fqnPrefix, int limitParam, String after) { + return tableDAO().listAfter(fqnPrefix, limitParam, after); + } + + @Override + public List listBefore(String fqnPrefix, int limitParam, String after) { + return tableDAO().listBefore(fqnPrefix, limitParam, after); + } + + @Override + public int listCount(String fqnPrefix) { + return tableDAO().listCount(fqnPrefix); + } + + @Override + public String getFullyQualifiedName(Table entity) { + return entity.getFullyQualifiedName(); + } + + @Override + public Table setFields(Table entity, Fields fields) throws IOException, ParseException { + return TableRepository.this.setFields(entity, fields); + } + + @Override + public ResultList
getResultList(List
entities, String beforeCursor, String afterCursor, int total) + throws GeneralSecurityException, UnsupportedEncodingException { + return new TableList(entities, beforeCursor, afterCursor, total); + } + }; + @Transaction - public TableList listAfter(Fields fields, String databaseFQN, int limitParam, String after) throws IOException, - ParseException, GeneralSecurityException { - // forward scrolling, if after == null then first page is being asked being asked - List jsons = tableDAO().listAfter(databaseFQN, limitParam + 1, after == null ? "" : - CipherText.instance().decrypt(after)); - - List
tables = new ArrayList<>(); - for (String json : jsons) { - tables.add(setFields(JsonUtils.readValue(json, Table.class), fields)); - } - int total = tableDAO().listCount(databaseFQN); - - String beforeCursor, afterCursor = null; - beforeCursor = after == null ? null : tables.get(0).getFullyQualifiedName(); - if (tables.size() > limitParam) { // If extra result exists, then next page exists - return after cursor - tables.remove(limitParam); - afterCursor = tables.get(limitParam - 1).getFullyQualifiedName(); - } - return new TableList(tables, beforeCursor, afterCursor, total); + public ResultList
listAfter(Fields fields, String databaseFQN, int limitParam, String after) + throws IOException, ParseException, GeneralSecurityException { + return EntityUtil.listAfter(entityRepository, Table.class, fields, databaseFQN, limitParam, after); } @Transaction - public TableList listBefore(Fields fields, String databaseFQN, int limitParam, String before) throws IOException, - ParseException, GeneralSecurityException { - // Reverse scrolling - Get one extra result used for computing before cursor - List jsons = tableDAO().listBefore(databaseFQN, limitParam + 1, CipherText.instance().decrypt(before)); - - List
tables = new ArrayList<>(); - for (String json : jsons) { - tables.add(setFields(JsonUtils.readValue(json, Table.class), fields)); - } - int total = tableDAO().listCount(databaseFQN); - - String beforeCursor = null, afterCursor; - if (tables.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor - tables.remove(0); - beforeCursor = tables.get(0).getFullyQualifiedName(); - } - afterCursor = tables.get(tables.size() - 1).getFullyQualifiedName(); - return new TableList(tables, beforeCursor, afterCursor, total); + public ResultList
listBefore(Fields fields, String databaseFQN, int limitParam, String before) + throws IOException, ParseException, GeneralSecurityException { + return EntityUtil.listBefore(entityRepository, Table.class, fields, databaseFQN, limitParam, before); } @Transaction diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java index 96d7ced7c77..6f06da479d8 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java @@ -130,31 +130,31 @@ public class TableResource { content = @Content(mediaType = "application/json", schema = @Schema(implementation = TableList.class))) }) - public TableList list(@Context UriInfo uriInfo, - @Context SecurityContext securityContext, - @Parameter(description = "Fields requested in the returned resource", + public ResultList
list(@Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Fields requested in the returned resource", schema = @Schema(type = "string", example = FIELDS)) @QueryParam("fields") String fieldsParam, - @Parameter(description = "Filter tables by database fully qualified name", + @Parameter(description = "Filter tables by database fully qualified name", schema = @Schema(type = "string", example = "snowflakeWestCoast.financeDB")) @QueryParam("database") String databaseParam, - @Parameter(description = "Limit the number tables returned. (1 to 1000000, default = 10) ", + @Parameter(description = "Limit the number tables returned. (1 to 1000000, default = 10) ", schema = @Schema(type = "string", example = "snowflakeWestCoast.financeDB")) @DefaultValue("10") @Min(1) @Max(1000000) @QueryParam("limit") int limitParam, - @Parameter(description = "Returns list of tables before this cursor", + @Parameter(description = "Returns list of tables before this cursor", schema = @Schema(type = "string")) @QueryParam("before") String before, - @Parameter(description = "Returns list of tables after this cursor", + @Parameter(description = "Returns list of tables after this cursor", schema = @Schema(type = "string")) @QueryParam("after") String after) throws IOException, ParseException, GeneralSecurityException { RestUtil.validateCursors(before, after); Fields fields = new Fields(FIELD_LIST, fieldsParam); - TableList tables; + ResultList
tables; if (before != null) { // Reverse paging tables = dao.listBefore(fields, databaseParam, limitParam, before); } else { // Forward paging or first page diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java index 9c4fb826da8..66d89271697 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java @@ -38,6 +38,7 @@ import org.openmetadata.catalog.exception.EntityNotFoundException; import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO; import org.openmetadata.catalog.jdbi3.DashboardRepository.DashboardDAO; import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseDAO; +import org.openmetadata.catalog.jdbi3.EntityRepository; import org.openmetadata.catalog.jdbi3.EntityRelationshipDAO; import org.openmetadata.catalog.jdbi3.MetricsRepository.MetricsDAO; import org.openmetadata.catalog.jdbi3.ModelRepository.ModelDAO; @@ -72,12 +73,15 @@ import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.type.TagLabel.LabelType; import org.openmetadata.catalog.type.UsageDetails; import org.openmetadata.catalog.type.UsageStats; +import org.openmetadata.common.utils.CipherText; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.UriInfo; import java.io.IOException; +import java.security.GeneralSecurityException; +import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -164,7 +168,7 @@ public final class EntityUtil { } else if (entity.equalsIgnoreCase(Entity.PIPELINE_SERVICE)) { PipelineServiceResource.addHref(uriInfo, ref); } else { - throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(ref.getType())); + throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(ref.getType())); } } @@ -185,7 +189,7 @@ public final class EntityUtil { if (ids.size() > 1) { LOG.warn("Possible database issues - multiple owners {} found for entity {}", ids, id); } - return ids.isEmpty() ? null : EntityUtil.populateOwner(userDAO, teamDAO, ids.get(0)); + return ids.isEmpty() ? null : EntityUtil.populateOwner(userDAO, teamDAO, ids.get(0)); } /** @@ -293,7 +297,7 @@ public final class EntityUtil { } else if (entity.equalsIgnoreCase(Entity.PIPELINE)) { Pipeline instance = EntityUtil.validate(id, pipelineDAO.findById(id), Pipeline.class); return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName()); - } else if (entity.equalsIgnoreCase(Entity.MODEL)) { + } else if (entity.equalsIgnoreCase(Entity.MODEL)) { Model instance = EntityUtil.validate(id, modelDAO.findById(id), Model.class); return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName()); } @@ -340,7 +344,7 @@ public final class EntityUtil { } else if (entity.equalsIgnoreCase(Entity.TASK)) { Task instance = EntityUtil.validate(fqn, taskDAO.findByFQN(fqn), Task.class); return getEntityReference(instance); - } else if (entity.equalsIgnoreCase(Entity.PIPELINE)) { + } else if (entity.equalsIgnoreCase(Entity.PIPELINE)) { Pipeline instance = EntityUtil.validate(fqn, pipelineDAO.findByFQN(fqn), Pipeline.class); return getEntityReference(instance); } else if (entity.equalsIgnoreCase(Entity.MODEL)) { @@ -431,7 +435,7 @@ public final class EntityUtil { throws IOException { String entityType = entityLink.getEntityType(); String fqn = entityLink.getEntityId(); - if(entityType.equalsIgnoreCase(Entity.USER)) { + if (entityType.equalsIgnoreCase(Entity.USER)) { return getEntityReference(EntityUtil.validate(fqn, userDAO.findByName(fqn), User.class)); } else if (entityType.equalsIgnoreCase(Entity.TEAM)) { return getEntityReference(EntityUtil.validate(fqn, teamDAO.findByName(fqn), Team.class)); @@ -650,4 +654,47 @@ public final class EntityUtil { return fieldList.contains(field); } } + + public static ResultList listAfter(EntityRepository dao, Class clz, Fields fields, String prefixFqn, + int limitParam, String after) + throws IOException, ParseException, GeneralSecurityException { + // forward scrolling, if after == null then first page is being asked being asked + List jsons = dao.listAfter(prefixFqn, limitParam + 1, after == null ? "" : + CipherText.instance().decrypt(after)); + + List entities = new ArrayList<>(); + for (String json : jsons) { + entities.add(dao.setFields(JsonUtils.readValue(json, clz), fields)); + } + int total = dao.listCount(prefixFqn); + + String beforeCursor, afterCursor = null; + beforeCursor = after == null ? null : dao.getFullyQualifiedName(entities.get(0)); + if (entities.size() > limitParam) { // If extra result exists, then next page exists - return after cursor + entities.remove(limitParam); + afterCursor = dao.getFullyQualifiedName(entities.get(limitParam - 1)); + } + return dao.getResultList(entities, beforeCursor, afterCursor, total); + } + + public static ResultList listBefore(EntityRepository dao, Class clz, Fields fields, String databaseFQN, + int limitParam, String before) + throws IOException, ParseException, GeneralSecurityException { + // Reverse scrolling - Get one extra result used for computing before cursor + List jsons = dao.listBefore(databaseFQN, limitParam + 1, CipherText.instance().decrypt(before)); + + List entities = new ArrayList<>(); + for (String json : jsons) { + entities.add(dao.setFields(JsonUtils.readValue(json, clz), fields)); + } + int total = dao.listCount(databaseFQN); + + String beforeCursor = null, afterCursor; + if (entities.size() > limitParam) { // If extra result exists, then previous page exists - return before cursor + entities.remove(0); + beforeCursor = dao.getFullyQualifiedName(entities.get(0)); + } + afterCursor = dao.getFullyQualifiedName(entities.get(entities.size() - 1)); + return dao.getResultList(entities, beforeCursor, afterCursor, total); + } }