yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-06 07:32:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

FIX: partitionColumn Migration (#15497)

Browse Source 
* fix: added try/catch + null fallback for getString() method of JsonObject instances

* style: ran java linting

* style: ran java linting

* fix: inverted conditional statement

* style: ran java linting

* fix: removed return and added else

---------

Co-authored-by: Ubuntu <ubuntu@ip-10-80-73-174.eu-west-3.compute.internal>
This commit is contained in:
Teddy 2024-03-14 07:28:52 +01:00 committed by GitHub
parent 8cc4ccdacd
commit 271acc4347
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 48 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v140/MigrationUtil.java
View File

@ -66,62 +66,63 @@ public class MigrationUtil {
}
private static void handleTablePartitionMigration(String jsonRow, CollectionDAO collectionDAO) {
JsonObject jsonObj = JsonUtils.readJson(jsonRow).asJsonObject();
try {
JsonObject jsonObj = JsonUtils.readJson(jsonRow).asJsonObject();
// We need to pop the tablePartition from the json before serializing it
JsonObject tablePartition = jsonObj.getJsonObject("tablePartition");
// We need to pop the tablePartition from the json before serializing it
JsonObject tablePartition = jsonObj.getJsonObject("tablePartition");
// Remove the tablePartition from the json. We need to convert it to a map to remove the key
// as JsonObject is immutable
HashMap<String, Object> jsonMap = JsonUtils.readValue(jsonObj.toString(), HashMap.class);
jsonMap.remove("tablePartition");
// Remove the tablePartition from the json. We need to convert it to a map to remove the key
// as JsonObject is immutable
HashMap<String, Object> jsonMap = JsonUtils.readValue(jsonObj.toString(), HashMap.class);
jsonMap.remove("tablePartition");
jsonObj = JsonUtils.readJson(JsonUtils.pojoToJson(jsonMap)).asJsonObject();
jsonObj = JsonUtils.readJson(JsonUtils.pojoToJson(jsonMap)).asJsonObject();
Table table = JsonUtils.readValue(jsonObj.toString(), Table.class);
Table table = JsonUtils.readValue(jsonObj.toString(), Table.class);
JsonArray partitionColumns = tablePartition.getJsonArray("columns");
if (tablePartition.isEmpty()) {
LOG.info("Table {} does not have partition details", table.getId());
return;
}
if (!tablePartition.isEmpty()) {
JsonArray partitionColumns = tablePartition.getJsonArray("columns");
List<PartitionColumnDetails> partitionColumnDetails = new ArrayList<PartitionColumnDetails>();
List<PartitionColumnDetails> partitionColumnDetails = new ArrayList<>();
if ((partitionColumns == null || partitionColumns.isEmpty())
&& table.getServiceType() == CreateDatabaseService.DatabaseServiceType.BigQuery) {
// BigQuery tables have pseudo columns for partitioning that were not being set in the
// partitionColumns entity
String interval = tablePartition.getString("interval");
if (interval != null) {
JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
switch (interval) {
case "HOUR" -> partitionColumns = jsonArrayBuilder.add("_PARTITIONTIME").build();
case "DAY" -> partitionColumns = jsonArrayBuilder.add("_PARTITIONDATE").build();
if ((partitionColumns == null || partitionColumns.isEmpty())
&& table.getServiceType() == CreateDatabaseService.DatabaseServiceType.BigQuery) {
// BigQuery tables can have pseudo columns for partitioning that were not being set in the
// partitionColumns entity
String interval = tablePartition.getString("interval", null);
if (interval != null) {
JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
switch (interval) {
case "HOUR" -> partitionColumns = jsonArrayBuilder.add("_PARTITIONTIME").build();
case "DAY" -> partitionColumns = jsonArrayBuilder.add("_PARTITIONDATE").build();
}
}
}
if (partitionColumns != null && !partitionColumns.isEmpty()) {
for (JsonValue column : partitionColumns) {
PartitionColumnDetails partitionColumnDetail = new PartitionColumnDetails();
partitionColumnDetail.setColumnName(((JsonString) column).getString());
String intervalType = tablePartition.getString("intervalType", null);
if (intervalType != null) {
partitionColumnDetail.setIntervalType(PartitionIntervalTypes.fromValue(intervalType));
}
partitionColumnDetail.setInterval(tablePartition.getString("interval", null));
partitionColumnDetails.add(partitionColumnDetail);
}
table.withTablePartition(new TablePartition().withColumns(partitionColumnDetails));
collectionDAO.tableDAO().update(table);
}
} else {
LOG.debug("Table {} does not have partition details", table.getId());
}
;
} catch (Exception exc) {
LOG.warn(
"Fail to migrate table partition. The partition detail may have been migrated already.");
LOG.debug(String.format("Table JSON %s\n", jsonRow), exc);
}
if (partitionColumns == null || partitionColumns.isEmpty()) {
throw new RuntimeException(
"tablePartition is not null but not column partition was defined for table "
+ table.getId());
}
for (JsonValue column : partitionColumns) {
PartitionColumnDetails partitionColumnDetail = new PartitionColumnDetails();
partitionColumnDetail.setColumnName(((JsonString) column).getString());
String intervalType = tablePartition.getString("intervalType");
if (intervalType != null) {
partitionColumnDetail.setIntervalType(PartitionIntervalTypes.fromValue(intervalType));
}
partitionColumnDetail.setInterval(tablePartition.getString("interval"));
partitionColumnDetails.add(partitionColumnDetail);
}
table.withTablePartition(new TablePartition().withColumns(partitionColumnDetails));
collectionDAO.tableDAO().update(table);
}
}