From 271acc43475b2052a79a0c2fd45af30dc233df9c Mon Sep 17 00:00:00 2001 From: Teddy Date: Thu, 14 Mar 2024 07:28:52 +0100 Subject: [PATCH] FIX: partitionColumn Migration (#15497) * fix: added try/catch + null fallback for getString() method of JsonObject instances * style: ran java linting * style: ran java linting * fix: inverted conditional statement * style: ran java linting * fix: removed return and added else --------- Co-authored-by: Ubuntu --- .../migration/utils/v140/MigrationUtil.java | 95 ++++++++++--------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v140/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v140/MigrationUtil.java index f5305aa34db..a5fbd05792b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v140/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v140/MigrationUtil.java @@ -66,62 +66,63 @@ public class MigrationUtil { } private static void handleTablePartitionMigration(String jsonRow, CollectionDAO collectionDAO) { - JsonObject jsonObj = JsonUtils.readJson(jsonRow).asJsonObject(); + try { + JsonObject jsonObj = JsonUtils.readJson(jsonRow).asJsonObject(); - // We need to pop the tablePartition from the json before serializing it - JsonObject tablePartition = jsonObj.getJsonObject("tablePartition"); + // We need to pop the tablePartition from the json before serializing it + JsonObject tablePartition = jsonObj.getJsonObject("tablePartition"); - // Remove the tablePartition from the json. We need to convert it to a map to remove the key - // as JsonObject is immutable - HashMap jsonMap = JsonUtils.readValue(jsonObj.toString(), HashMap.class); - jsonMap.remove("tablePartition"); + // Remove the tablePartition from the json. We need to convert it to a map to remove the key + // as JsonObject is immutable + HashMap jsonMap = JsonUtils.readValue(jsonObj.toString(), HashMap.class); + jsonMap.remove("tablePartition"); - jsonObj = JsonUtils.readJson(JsonUtils.pojoToJson(jsonMap)).asJsonObject(); + jsonObj = JsonUtils.readJson(JsonUtils.pojoToJson(jsonMap)).asJsonObject(); - Table table = JsonUtils.readValue(jsonObj.toString(), Table.class); + Table table = JsonUtils.readValue(jsonObj.toString(), Table.class); - JsonArray partitionColumns = tablePartition.getJsonArray("columns"); - if (tablePartition.isEmpty()) { - LOG.info("Table {} does not have partition details", table.getId()); - return; - } + if (!tablePartition.isEmpty()) { + JsonArray partitionColumns = tablePartition.getJsonArray("columns"); - List partitionColumnDetails = new ArrayList(); + List partitionColumnDetails = new ArrayList<>(); - if ((partitionColumns == null || partitionColumns.isEmpty()) - && table.getServiceType() == CreateDatabaseService.DatabaseServiceType.BigQuery) { - // BigQuery tables have pseudo columns for partitioning that were not being set in the - // partitionColumns entity - String interval = tablePartition.getString("interval"); - if (interval != null) { - JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); - switch (interval) { - case "HOUR" -> partitionColumns = jsonArrayBuilder.add("_PARTITIONTIME").build(); - case "DAY" -> partitionColumns = jsonArrayBuilder.add("_PARTITIONDATE").build(); + if ((partitionColumns == null || partitionColumns.isEmpty()) + && table.getServiceType() == CreateDatabaseService.DatabaseServiceType.BigQuery) { + // BigQuery tables can have pseudo columns for partitioning that were not being set in the + // partitionColumns entity + String interval = tablePartition.getString("interval", null); + if (interval != null) { + JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); + switch (interval) { + case "HOUR" -> partitionColumns = jsonArrayBuilder.add("_PARTITIONTIME").build(); + case "DAY" -> partitionColumns = jsonArrayBuilder.add("_PARTITIONDATE").build(); + } + } } + + if (partitionColumns != null && !partitionColumns.isEmpty()) { + for (JsonValue column : partitionColumns) { + PartitionColumnDetails partitionColumnDetail = new PartitionColumnDetails(); + partitionColumnDetail.setColumnName(((JsonString) column).getString()); + String intervalType = tablePartition.getString("intervalType", null); + if (intervalType != null) { + partitionColumnDetail.setIntervalType(PartitionIntervalTypes.fromValue(intervalType)); + } + partitionColumnDetail.setInterval(tablePartition.getString("interval", null)); + partitionColumnDetails.add(partitionColumnDetail); + } + + table.withTablePartition(new TablePartition().withColumns(partitionColumnDetails)); + + collectionDAO.tableDAO().update(table); + } + } else { + LOG.debug("Table {} does not have partition details", table.getId()); } - ; + } catch (Exception exc) { + LOG.warn( + "Fail to migrate table partition. The partition detail may have been migrated already."); + LOG.debug(String.format("Table JSON %s\n", jsonRow), exc); } - - if (partitionColumns == null || partitionColumns.isEmpty()) { - throw new RuntimeException( - "tablePartition is not null but not column partition was defined for table " - + table.getId()); - } - - for (JsonValue column : partitionColumns) { - PartitionColumnDetails partitionColumnDetail = new PartitionColumnDetails(); - partitionColumnDetail.setColumnName(((JsonString) column).getString()); - String intervalType = tablePartition.getString("intervalType"); - if (intervalType != null) { - partitionColumnDetail.setIntervalType(PartitionIntervalTypes.fromValue(intervalType)); - } - partitionColumnDetail.setInterval(tablePartition.getString("interval")); - partitionColumnDetails.add(partitionColumnDetail); - } - - table.withTablePartition(new TablePartition().withColumns(partitionColumnDetails)); - - collectionDAO.tableDAO().update(table); } }