diff --git a/bootstrap/bootstrap_storage.sh b/bootstrap/bootstrap_storage.sh index 6712c1b855d..9680f534335 100755 --- a/bootstrap/bootstrap_storage.sh +++ b/bootstrap/bootstrap_storage.sh @@ -88,6 +88,9 @@ drop-create ) drop-create-all ) execute "drop" && execute "create" && execute "es-drop" && execute "es-create" ;; +migrate-all ) + execute "migrate" && execute "es-migrate" + ;; *) printUsage exit 1 diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java index bc80809a673..2ea3a5e096f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java @@ -27,6 +27,7 @@ import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.common.xcontent.XContentType; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.entity.data.Dashboard; @@ -84,6 +85,16 @@ public class ElasticSearchIndexDefinition { } } + public void updateIndexes() { + try { + for (ElasticSearchIndexType elasticSearchIndexType : ElasticSearchIndexType.values()) { + updateIndex(elasticSearchIndexType); + } + } catch (Exception e) { + LOG.error("Failed to created Elastic Search indexes due to", e); + } + } + public void dropIndexes() { try { for (ElasticSearchIndexType elasticSearchIndexType : ElasticSearchIndexType.values()) { @@ -123,6 +134,32 @@ public class ElasticSearchIndexDefinition { return true; } + private boolean updateIndex(ElasticSearchIndexType elasticSearchIndexType) { + try { + GetIndexRequest gRequest = new GetIndexRequest(elasticSearchIndexType.indexName); + gRequest.local(false); + boolean exists = client.indices().exists(gRequest, RequestOptions.DEFAULT); + String elasticSearchIndexMapping = getIndexMapping(elasticSearchIndexType); + if (exists) { + PutMappingRequest request = new PutMappingRequest(elasticSearchIndexType.indexName); + request.source(elasticSearchIndexMapping, XContentType.JSON); + AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT); + LOG.info(elasticSearchIndexType.indexName + " Updated " + putMappingResponse.isAcknowledged()); + } else { + CreateIndexRequest request = new CreateIndexRequest(elasticSearchIndexType.indexName); + request.mapping(elasticSearchIndexMapping, XContentType.JSON); + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); + LOG.info(elasticSearchIndexType.indexName + " Created " + createIndexResponse.isAcknowledged()); + } + setIndexStatus(elasticSearchIndexType, ElasticSearchIndexStatus.CREATED); + } catch (Exception e) { + setIndexStatus(elasticSearchIndexType, ElasticSearchIndexStatus.FAILED); + LOG.error("Failed to created Elastic Search indexes due to", e); + return false; + } + return true; + } + private boolean deleteIndex(ElasticSearchIndexType elasticSearchIndexType) { try { DeleteIndexRequest request = new DeleteIndexRequest(elasticSearchIndexType.indexName); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/TablesInitializer.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/TablesInitializer.java index c8cbca3a3ea..b94bab93027 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/TablesInitializer.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/TablesInitializer.java @@ -76,6 +76,7 @@ public final class TablesInitializer { null, SchemaMigrationOption.ES_CREATE.toString(), false, "Creates all the indexes in the elastic search"); OPTIONS.addOption( null, SchemaMigrationOption.ES_DROP.toString(), false, "Drop all the indexes in the elastic search"); + OPTIONS.addOption(null, SchemaMigrationOption.ES_MIGRATE.toString(), false, "Update Elastic Search index mapping"); } private TablesInitializer() {} @@ -207,6 +208,10 @@ public final class TablesInitializer { esIndexDefinition = new ElasticSearchIndexDefinition(client); esIndexDefinition.createIndexes(); break; + case ES_MIGRATE: + esIndexDefinition = new ElasticSearchIndexDefinition(client); + esIndexDefinition.updateIndexes(); + break; case ES_DROP: esIndexDefinition = new ElasticSearchIndexDefinition(client); esIndexDefinition.dropIndexes(); @@ -230,7 +235,8 @@ public final class TablesInitializer { DROP("drop"), REPAIR("repair"), ES_DROP("es-drop"), - ES_CREATE("es-create"); + ES_CREATE("es-create"), + ES_MIGRATE("es-migrate"); private final String value; diff --git a/docker/metadata/openmetadata-start.sh b/docker/metadata/openmetadata-start.sh index 66ca998ad83..19821a7d47b 100644 --- a/docker/metadata/openmetadata-start.sh +++ b/docker/metadata/openmetadata-start.sh @@ -13,5 +13,5 @@ while ! wget -O /dev/null -o /dev/null mysql:3306; do sleep 5; done cp /openmetadata.yaml /openmetadata-*/conf/openmetadata.yaml cd /openmetadata-*/ -./bootstrap/bootstrap_storage.sh drop-create-all +./bootstrap/bootstrap_storage.sh migrate-all ./bin/openmetadata-server-start.sh conf/openmetadata.yaml