yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-06 15:43:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

FIX: partitionColumn Migration (#15497)

Browse Source 
* fix: added try/catch + null fallback for getString() method of JsonObject instances

* style: ran java linting

* style: ran java linting

* fix: inverted conditional statement

* style: ran java linting

* fix: removed return and added else

---------

Co-authored-by: Ubuntu <ubuntu@ip-10-80-73-174.eu-west-3.compute.internal>
This commit is contained in:
Teddy 2024-03-14 07:28:52 +01:00 committed by GitHub
parent 8cc4ccdacd
commit 271acc4347
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 48 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v140/MigrationUtil.java
View File

@ -66,6 +66,7 @@ public class MigrationUtil {
} }
private static void handleTablePartitionMigration(String jsonRow, CollectionDAO collectionDAO) { private static void handleTablePartitionMigration(String jsonRow, CollectionDAO collectionDAO) {
try {
JsonObject jsonObj = JsonUtils.readJson(jsonRow).asJsonObject(); JsonObject jsonObj = JsonUtils.readJson(jsonRow).asJsonObject();
// We need to pop the tablePartition from the json before serializing it // We need to pop the tablePartition from the json before serializing it
@ -80,19 +81,16 @@ public class MigrationUtil {
Table table = JsonUtils.readValue(jsonObj.toString(), Table.class); Table table = JsonUtils.readValue(jsonObj.toString(), Table.class);
if (!tablePartition.isEmpty()) {
JsonArray partitionColumns = tablePartition.getJsonArray("columns"); JsonArray partitionColumns = tablePartition.getJsonArray("columns");
if (tablePartition.isEmpty()) {
LOG.info("Table {} does not have partition details", table.getId());
return;
}
List<PartitionColumnDetails> partitionColumnDetails = new ArrayList<PartitionColumnDetails>(); List<PartitionColumnDetails> partitionColumnDetails = new ArrayList<>();
if ((partitionColumns == null || partitionColumns.isEmpty()) if ((partitionColumns == null || partitionColumns.isEmpty())
&& table.getServiceType() == CreateDatabaseService.DatabaseServiceType.BigQuery) { && table.getServiceType() == CreateDatabaseService.DatabaseServiceType.BigQuery) {
// BigQuery tables have pseudo columns for partitioning that were not being set in the // BigQuery tables can have pseudo columns for partitioning that were not being set in the
// partitionColumns entity // partitionColumns entity
String interval = tablePartition.getString("interval"); String interval = tablePartition.getString("interval", null);
if (interval != null) { if (interval != null) {
JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
switch (interval) { switch (interval) {
@ -100,23 +98,17 @@ public class MigrationUtil {
case "DAY" -> partitionColumns = jsonArrayBuilder.add("_PARTITIONDATE").build(); case "DAY" -> partitionColumns = jsonArrayBuilder.add("_PARTITIONDATE").build();
} }
} }
;
}
if (partitionColumns == null || partitionColumns.isEmpty()) {
throw new RuntimeException(
"tablePartition is not null but not column partition was defined for table "
+ table.getId());
} }
if (partitionColumns != null && !partitionColumns.isEmpty()) {
for (JsonValue column : partitionColumns) { for (JsonValue column : partitionColumns) {
PartitionColumnDetails partitionColumnDetail = new PartitionColumnDetails(); PartitionColumnDetails partitionColumnDetail = new PartitionColumnDetails();
partitionColumnDetail.setColumnName(((JsonString) column).getString()); partitionColumnDetail.setColumnName(((JsonString) column).getString());
String intervalType = tablePartition.getString("intervalType"); String intervalType = tablePartition.getString("intervalType", null);
if (intervalType != null) { if (intervalType != null) {
partitionColumnDetail.setIntervalType(PartitionIntervalTypes.fromValue(intervalType)); partitionColumnDetail.setIntervalType(PartitionIntervalTypes.fromValue(intervalType));
} }
partitionColumnDetail.setInterval(tablePartition.getString("interval")); partitionColumnDetail.setInterval(tablePartition.getString("interval", null));
partitionColumnDetails.add(partitionColumnDetail); partitionColumnDetails.add(partitionColumnDetail);
} }
@ -124,4 +116,13 @@ public class MigrationUtil {
collectionDAO.tableDAO().update(table); collectionDAO.tableDAO().update(table);
} }
} else {
LOG.debug("Table {} does not have partition details", table.getId());
}
} catch (Exception exc) {
LOG.warn(
"Fail to migrate table partition. The partition detail may have been migrated already.");
LOG.debug(String.format("Table JSON %s\n", jsonRow), exc);
}
}
} }