yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-12 18:47:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(restoreIndices): fix bug in urn paginated restoreIndices exit code (#11443)

Browse Source
This commit is contained in:
david-leifker 2024-09-20 10:33:37 -05:00 committed by GitHub
parent b607a66c05
commit f6dde2bf03
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 35 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
datahub-upgrade/build.gradle
View File

@ -121,6 +121,34 @@ task run(type: Exec) {
"-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdate"
}
/**
* Runs RestoreIndices on locally running system. The batchSize are set to
* test the process with pagination and not designed for optimal performance.
*/
task runRestoreIndices(type: Exec) {
dependsOn bootJar
group = "Execution"
description = "Run the restore indices process locally."
environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
"-jar",
"-Dkafka.schemaRegistry.url=http://localhost:8080/schema-registry/api",
"-Dserver.port=8083",
bootJar.getArchiveFile().get(), "-u", "RestoreIndices", "-a", "batchSize=100"
}
task runRestoreIndicesUrn(type: Exec) {
dependsOn bootJar
group = "Execution"
description = "Run the restore indices process locally."
environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
"-jar",
"-Dkafka.schemaRegistry.url=http://localhost:8080/schema-registry/api",
"-Dserver.port=8083",
bootJar.getArchiveFile().get(), "-u", "RestoreIndices", "-a", "batchSize=100", "-a", "urnBasedPagination=true"
}
docker {
name "${docker_registry}/${docker_repo}:v${version}"
version "v${version}"

6
datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java
View File

@ -16,6 +16,7 @@ import io.ebean.ExpressionList;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@ -189,9 +190,14 @@ public class SendMAEStep implements UpgradeStep {
context.report().addLine(String.format("Rows processed this loop %d", rowsProcessed));
start += args.batchSize;
} catch (InterruptedException | ExecutionException e) {
if (e.getCause() instanceof NoSuchElementException) {
context.report().addLine("End of data.");
break;
} else {
return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED);
}
}
}
} else {
while (start < rowCount) {
args = args.clone();