yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-30 08:57:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migration error in listing by Name because of duplicate name

Browse Source
This commit is contained in:
mohitdeuex 2023-07-03 22:57:36 +05:30
parent 989914f84e
commit 89b0de4eac
2 changed files with 21 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java
View File

@ -224,6 +224,16 @@ public interface EntityDAO<T extends EntityInterface> {
@Bind("limit") int limit,
@Bind("after") String after);
@SqlQuery(
"SELECT json FROM <table> <cond> AND " + "<orderByColumn> > :after " + "ORDER BY :orderBy " + "LIMIT :limit")
List<String> listAfterWithOrderBy(
@Define("table") String table,
@Define("orderByColumn") String orderByColumn,
@Define("cond") String cond,
@Bind("limit") int limit,
@Bind("after") String after,
@Bind("orderBy") String orderBy);
@SqlQuery("SELECT json FROM <table> <cond> AND " + "ORDER BY <nameColumn> " + "LIMIT :limit " + "OFFSET :offset")
List<String> listAfter(
@Define("table") String table,
@ -343,6 +353,12 @@ public interface EntityDAO<T extends EntityInterface> {
return listAfter(getTableName(), getNameColumn(), filter.getCondition(), limit, after);
}
default List<String> listAfterWithOrderBy(ListFilter filter, int limit, String after, String orderBy) {
// This is based on field not fqn or name
// Ordering and Paginating on name or fqn should be done using above function as requires unquoting/quoting
return listAfterWithOrderBy(getTableName(), orderBy, filter.getCondition(), limit, after, orderBy);
}
default List<String> listAfter(ListFilter filter, int limit, int offset) {
return listAfter(getTableName(), getNameHashColumn(), filter.getCondition(), limit, offset);
}

8
openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationUtil.java
View File

@ -130,13 +130,13 @@ public class MigrationUtil {
ListFilter filter = new ListFilter(Include.ALL);
List<T> entities;
String after = null;
PreparedBatch upsertBatch = handle.prepareBatch(updateSql);
do {
PreparedBatch upsertBatch = handle.prepareBatch(updateSql);
// Create empty Array
entities = new ArrayList<>();
// Read from Database
List<String> jsons = dao.listAfter(filter, limitParam + 1, after == null ? "" : after);
List<String> jsons = dao.listAfterWithOrderBy(filter, limitParam + 1, after == null ? "" : after, "id");
for (String json : jsons) {
T entity = JsonUtils.readValue(json, clazz);
entities.add(entity);
@ -144,7 +144,7 @@ public class MigrationUtil {
String afterCursor = null;
if (entities.size() > limitParam) {
entities.remove(limitParam);
afterCursor = entities.get(limitParam - 1).getName();
afterCursor = entities.get(limitParam - 1).getId().toString();
}
after = afterCursor;
@ -299,6 +299,8 @@ public class MigrationUtil {
if (total > 10000) {
upsertBatch.execute();
total = 0;
// Creating a new batch result in faster writes
upsertBatch = handle.prepareBatch(updateSql);
}
}
}