yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-07 05:53:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Change ignoreFileChksum to force

Browse Source
This commit is contained in:
mohitdeuex 2023-07-26 16:40:52 +05:30
parent 72e4d0070f
commit 8ab4072504
3 changed files with 22 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bootstrap/bootstrap_storage.sh
View File

@ -13,12 +13,12 @@
# Resolve links - $0 may be a softlink # Resolve links - $0 may be a softlink
PRG="${0}" PRG="${0}"
debug="$2" debug="$2"
ignoreFileChecksum="$3" force=false
if [ -n "${ignoreFileChecksum+x}" ]; then if [ -z "$3" ]; then
ignoreFileChecksum=true force=false
else else
ignoreFileChecksum=false force=true
fi fi
while [ -h "${PRG}" ]; do while [ -h "${PRG}" ]; do
@ -60,12 +60,12 @@ execute() {
if [ ${debug} ] ; then if [ ${debug} ] ; then
echo "Using Configuration file: ${CONFIG_FILE_PATH}" echo "Using Configuration file: ${CONFIG_FILE_PATH}"
fi fi
${JAVA} -Dbootstrap.dir=$BOOTSTRAP_DIR -cp ${CLASSPATH} ${TABLE_INITIALIZER_MAIN_CLASS} -c ${CONFIG_FILE_PATH} -s ${SCRIPT_ROOT_DIR} --${1} -ignoreCheckSum ${ignoreFileChecksum} -${debug} ${JAVA} -Dbootstrap.dir=$BOOTSTRAP_DIR -cp ${CLASSPATH} ${TABLE_INITIALIZER_MAIN_CLASS} -c ${CONFIG_FILE_PATH} -s ${SCRIPT_ROOT_DIR} --${1} -force ${force} -${debug}
} }
printUsage() { printUsage() {
cat <<-EOF cat <<-EOF
USAGE: $0 [create|migrate|info|validate|drop|drop-create|es-drop|es-create|drop-create-all|migrate-all|repair|check-connection|rotate] [debug] [ignoreFileChecksum] USAGE: $0 [create|migrate|info|validate|drop|drop-create|es-drop|es-create|drop-create-all|migrate-all|repair|check-connection|rotate] [debug] [force]
create : Creates the tables. The target database should be empty create : Creates the tables. The target database should be empty
migrate : Migrates the database to the latest version or creates the tables if the database is empty. Use "info" to see the current version and the pending migrations migrate : Migrates the database to the latest version or creates the tables if the database is empty. Use "info" to see the current version and the pending migrations
info : Shows the list of migrations applied and the pending migration waiting to be applied on the target database info : Shows the list of migrations applied and the pending migration waiting to be applied on the target database
@ -81,7 +81,7 @@ USAGE: $0 [create|migrate|info|validate|drop|drop-create|es-drop|es-create|drop-
check-connection : Checks if a connection can be successfully obtained for the target database check-connection : Checks if a connection can be successfully obtained for the target database
rotate : Rotate the Fernet Key defined in $FERNET_KEY rotate : Rotate the Fernet Key defined in $FERNET_KEY
debug : Enable Debugging Mode to get more info debug : Enable Debugging Mode to get more info
ignoreFileChecksum : Ignore Checksum forces Server Migration for already run migration to be run again force : Forces the server Migration to be ran, even if already ran
EOF EOF
} }

8
openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationWorkflow.java
View File

@ -15,12 +15,12 @@ public class MigrationWorkflow {
private final List<MigrationStep> migrations; private final List<MigrationStep> migrations;
private final MigrationDAO migrationDAO; private final MigrationDAO migrationDAO;
private final Jdbi jdbi; private final Jdbi jdbi;
private boolean ignoreFileChecksum = false; private final boolean forceMigrations;
public MigrationWorkflow(Jdbi jdbi, List<MigrationStep> migrationSteps, boolean ignoreFileChecksum) { public MigrationWorkflow(Jdbi jdbi, List<MigrationStep> migrationSteps, boolean forceMigrations) {
this.jdbi = jdbi; this.jdbi = jdbi;
this.migrationDAO = jdbi.onDemand(MigrationDAO.class); this.migrationDAO = jdbi.onDemand(MigrationDAO.class);
this.ignoreFileChecksum = ignoreFileChecksum; this.forceMigrations = forceMigrations;
// Sort Migration on the basis of version // Sort Migration on the basis of version
migrationSteps.sort(Comparator.comparing(MigrationStep::getMigrationVersion)); migrationSteps.sort(Comparator.comparing(MigrationStep::getMigrationVersion));
@ -62,7 +62,7 @@ public class MigrationWorkflow {
if (maxMigration.compareTo(step.getMigrationVersion()) < 0) { if (maxMigration.compareTo(step.getMigrationVersion()) < 0) {
// This a new Step file // This a new Step file
result.add(step); result.add(step);
} else if (ignoreFileChecksum || !checksum.equals(step.getFileUuid())) { } else if (forceMigrations || !checksum.equals(step.getFileUuid())) {
// This migration step was ran already, if checksum is equal this step can be ignored // This migration step was ran already, if checksum is equal this step can be ignored
LOG.warn( LOG.warn(
"[Migration Workflow] You are changing an older Migration File. This is not advised. Add your changes to latest Migrations."); "[Migration Workflow] You are changing an older Migration File. This is not advised. Add your changes to latest Migrations.");

23
openmetadata-service/src/main/java/org/openmetadata/service/util/TablesInitializer.java
View File

@ -64,11 +64,11 @@ public final class TablesInitializer {
private static final String DEBUG_MODE_ENABLED = "debug_mode"; private static final String DEBUG_MODE_ENABLED = "debug_mode";
private static final String OPTION_SCRIPT_ROOT_PATH = "script-root"; private static final String OPTION_SCRIPT_ROOT_PATH = "script-root";
private static final String OPTION_CONFIG_FILE_PATH = "config"; private static final String OPTION_CONFIG_FILE_PATH = "config";
private static final String OPTION_IGNORE_SERVER_FILE_CHECKSUM = "ignoreCheckSum"; private static final String OPTION_FORCE_MIGRATIONS = "force";
private static final String DISABLE_VALIDATE_ON_MIGRATE = "disable-validate-on-migrate"; private static final String DISABLE_VALIDATE_ON_MIGRATE = "disable-validate-on-migrate";
private static final Options OPTIONS; private static final Options OPTIONS;
private static boolean debugMode = false; private static boolean debugMode = false;
private static boolean ignoreServerFileChecksum = false; private static boolean forceMigrations = false;
static { static {
OPTIONS = new Options(); OPTIONS = new Options();
@ -76,10 +76,10 @@ public final class TablesInitializer {
OPTIONS.addOption("s", OPTION_SCRIPT_ROOT_PATH, true, "Root directory of script path"); OPTIONS.addOption("s", OPTION_SCRIPT_ROOT_PATH, true, "Root directory of script path");
OPTIONS.addOption("c", OPTION_CONFIG_FILE_PATH, true, "Config file path"); OPTIONS.addOption("c", OPTION_CONFIG_FILE_PATH, true, "Config file path");
OPTIONS.addOption( OPTIONS.addOption(
"ignoreCheckSum", OPTION_FORCE_MIGRATIONS,
OPTION_IGNORE_SERVER_FILE_CHECKSUM, OPTION_FORCE_MIGRATIONS,
true, true,
"Ignore the server checksum and rerun same file in migrate"); "Ignore the server checksum and force migrations to be run again");
OPTIONS.addOption(null, SchemaMigrationOption.CREATE.toString(), false, "Run sql migrations from scratch"); OPTIONS.addOption(null, SchemaMigrationOption.CREATE.toString(), false, "Run sql migrations from scratch");
OPTIONS.addOption(null, SchemaMigrationOption.DROP.toString(), false, "Drop all the tables in the target database"); OPTIONS.addOption(null, SchemaMigrationOption.DROP.toString(), false, "Drop all the tables in the target database");
OPTIONS.addOption( OPTIONS.addOption(
@ -126,8 +126,8 @@ public final class TablesInitializer {
if (commandLine.hasOption(DEBUG_MODE_ENABLED)) { if (commandLine.hasOption(DEBUG_MODE_ENABLED)) {
debugMode = true; debugMode = true;
} }
if (commandLine.hasOption(OPTION_IGNORE_SERVER_FILE_CHECKSUM)) { if (commandLine.hasOption(OPTION_FORCE_MIGRATIONS)) {
ignoreServerFileChecksum = Boolean.parseBoolean(commandLine.getOptionValue(OPTION_IGNORE_SERVER_FILE_CHECKSUM)); forceMigrations = Boolean.parseBoolean(commandLine.getOptionValue(OPTION_FORCE_MIGRATIONS));
} }
boolean isSchemaMigrationOptionSpecified = false; boolean isSchemaMigrationOptionSpecified = false;
SchemaMigrationOption schemaMigrationOptionSpecified = null; SchemaMigrationOption schemaMigrationOptionSpecified = null;
@ -275,13 +275,13 @@ public final class TablesInitializer {
} }
flyway.migrate(); flyway.migrate();
validateAndRunSystemDataMigrations( validateAndRunSystemDataMigrations(
jdbi, ConnectionType.from(config.getDataSourceFactory().getDriverClass()), ignoreServerFileChecksum); jdbi, ConnectionType.from(config.getDataSourceFactory().getDriverClass()), forceMigrations);
break; break;
case MIGRATE: case MIGRATE:
flyway.migrate(); flyway.migrate();
// Validate and Run System Data Migrations // Validate and Run System Data Migrations
validateAndRunSystemDataMigrations( validateAndRunSystemDataMigrations(
jdbi, ConnectionType.from(config.getDataSourceFactory().getDriverClass()), ignoreServerFileChecksum); jdbi, ConnectionType.from(config.getDataSourceFactory().getDriverClass()), forceMigrations);
break; break;
case INFO: case INFO:
printToConsoleMandatory(dumpToAsciiTable(flyway.info().all())); printToConsoleMandatory(dumpToAsciiTable(flyway.info().all()));
@ -331,11 +331,10 @@ public final class TablesInitializer {
} }
} }
public static void validateAndRunSystemDataMigrations( public static void validateAndRunSystemDataMigrations(Jdbi jdbi, ConnectionType connType, boolean forceMigrations) {
Jdbi jdbi, ConnectionType connType, boolean ignoreFileChecksum) {
DatasourceConfig.initialize(connType.label); DatasourceConfig.initialize(connType.label);
List<MigrationStep> loadedMigrationFiles = getServerMigrationFiles(connType); List<MigrationStep> loadedMigrationFiles = getServerMigrationFiles(connType);
MigrationWorkflow workflow = new MigrationWorkflow(jdbi, loadedMigrationFiles, ignoreFileChecksum); MigrationWorkflow workflow = new MigrationWorkflow(jdbi, loadedMigrationFiles, forceMigrations);
workflow.runMigrationWorkflows(); workflow.runMigrationWorkflows();
} }