diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java index 98bcfe14231..ef8e990bf53 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java @@ -224,6 +224,16 @@ public interface EntityDAO { @Bind("limit") int limit, @Bind("after") String after); + @SqlQuery( + "SELECT json FROM AND " + " > :after " + "ORDER BY :orderBy " + "LIMIT :limit") + List listAfterWithOrderBy( + @Define("table") String table, + @Define("orderByColumn") String orderByColumn, + @Define("cond") String cond, + @Bind("limit") int limit, + @Bind("after") String after, + @Bind("orderBy") String orderBy); + @SqlQuery("SELECT json FROM
AND " + "ORDER BY " + "LIMIT :limit " + "OFFSET :offset") List listAfter( @Define("table") String table, @@ -343,6 +353,12 @@ public interface EntityDAO { return listAfter(getTableName(), getNameColumn(), filter.getCondition(), limit, after); } + default List listAfterWithOrderBy(ListFilter filter, int limit, String after, String orderBy) { + // This is based on field not fqn or name + // Ordering and Paginating on name or fqn should be done using above function as requires unquoting/quoting + return listAfterWithOrderBy(getTableName(), orderBy, filter.getCondition(), limit, after, orderBy); + } + default List listAfter(ListFilter filter, int limit, int offset) { return listAfter(getTableName(), getNameHashColumn(), filter.getCondition(), limit, offset); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationUtil.java index 4a23b60a2b0..b663da832ce 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationUtil.java @@ -130,13 +130,13 @@ public class MigrationUtil { ListFilter filter = new ListFilter(Include.ALL); List entities; String after = null; - PreparedBatch upsertBatch = handle.prepareBatch(updateSql); do { + PreparedBatch upsertBatch = handle.prepareBatch(updateSql); // Create empty Array entities = new ArrayList<>(); // Read from Database - List jsons = dao.listAfter(filter, limitParam + 1, after == null ? "" : after); + List jsons = dao.listAfterWithOrderBy(filter, limitParam + 1, after == null ? "" : after, "id"); for (String json : jsons) { T entity = JsonUtils.readValue(json, clazz); entities.add(entity); @@ -144,7 +144,7 @@ public class MigrationUtil { String afterCursor = null; if (entities.size() > limitParam) { entities.remove(limitParam); - afterCursor = entities.get(limitParam - 1).getName(); + afterCursor = entities.get(limitParam - 1).getId().toString(); } after = afterCursor; @@ -299,6 +299,8 @@ public class MigrationUtil { if (total > 10000) { upsertBatch.execute(); total = 0; + // Creating a new batch result in faster writes + upsertBatch = handle.prepareBatch(updateSql); } } }