Fix #1814: Add migrate-all option to update ElasticSearch index mappings (#1815)

* Fix #1814: Add migrate-all option to update ElasticSearch index mappings

* Fix #1814: Add migrate-all option to update ElasticSearch index mappings

bootstrap/bootstrap_storage.sh

@@ -88,6 +88,9 @@ drop-create )
 drop-create-all )
     execute "drop" && execute "create" && execute "es-drop" && execute "es-create"
     ;;
+migrate-all )
+    execute "migrate" && execute "es-migrate"
+    ;;
 *)
     printUsage
     exit 1

catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java

@@ -27,6 +27,7 @@ import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.client.indices.CreateIndexResponse;
 import org.elasticsearch.client.indices.GetIndexRequest;
+import org.elasticsearch.client.indices.PutMappingRequest;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.entity.data.Dashboard;
@@ -84,6 +85,16 @@ public class ElasticSearchIndexDefinition {
     }
   }
 
+  public void updateIndexes() {
+    try {
+      for (ElasticSearchIndexType elasticSearchIndexType : ElasticSearchIndexType.values()) {
+        updateIndex(elasticSearchIndexType);
+      }
+    } catch (Exception e) {
+      LOG.error("Failed to created Elastic Search indexes due to", e);
+    }
+  }
+
   public void dropIndexes() {
     try {
       for (ElasticSearchIndexType elasticSearchIndexType : ElasticSearchIndexType.values()) {
@@ -123,6 +134,32 @@ public class ElasticSearchIndexDefinition {
     return true;
   }
 
+  private boolean updateIndex(ElasticSearchIndexType elasticSearchIndexType) {
+    try {
+      GetIndexRequest gRequest = new GetIndexRequest(elasticSearchIndexType.indexName);
+      gRequest.local(false);
+      boolean exists = client.indices().exists(gRequest, RequestOptions.DEFAULT);
+      String elasticSearchIndexMapping = getIndexMapping(elasticSearchIndexType);
+      if (exists) {
+        PutMappingRequest request = new PutMappingRequest(elasticSearchIndexType.indexName);
+        request.source(elasticSearchIndexMapping, XContentType.JSON);
+        AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
+        LOG.info(elasticSearchIndexType.indexName + " Updated " + putMappingResponse.isAcknowledged());
+      } else {
+        CreateIndexRequest request = new CreateIndexRequest(elasticSearchIndexType.indexName);
+        request.mapping(elasticSearchIndexMapping, XContentType.JSON);
+        CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT);
+        LOG.info(elasticSearchIndexType.indexName + " Created " + createIndexResponse.isAcknowledged());
+      }
+      setIndexStatus(elasticSearchIndexType, ElasticSearchIndexStatus.CREATED);
+    } catch (Exception e) {
+      setIndexStatus(elasticSearchIndexType, ElasticSearchIndexStatus.FAILED);
+      LOG.error("Failed to created Elastic Search indexes due to", e);
+      return false;
+    }
+    return true;
+  }
+
   private boolean deleteIndex(ElasticSearchIndexType elasticSearchIndexType) {
     try {
       DeleteIndexRequest request = new DeleteIndexRequest(elasticSearchIndexType.indexName);

catalog-rest-service/src/main/java/org/openmetadata/catalog/util/TablesInitializer.java

@@ -76,6 +76,7 @@ public final class TablesInitializer {
         null, SchemaMigrationOption.ES_CREATE.toString(), false, "Creates all the indexes in the elastic search");
     OPTIONS.addOption(
         null, SchemaMigrationOption.ES_DROP.toString(), false, "Drop all the indexes in the elastic search");
+    OPTIONS.addOption(null, SchemaMigrationOption.ES_MIGRATE.toString(), false, "Update Elastic Search index mapping");
   }
 
   private TablesInitializer() {}
@@ -207,6 +208,10 @@ public final class TablesInitializer {
         esIndexDefinition = new ElasticSearchIndexDefinition(client);
         esIndexDefinition.createIndexes();
         break;
+      case ES_MIGRATE:
+        esIndexDefinition = new ElasticSearchIndexDefinition(client);
+        esIndexDefinition.updateIndexes();
+        break;
       case ES_DROP:
         esIndexDefinition = new ElasticSearchIndexDefinition(client);
         esIndexDefinition.dropIndexes();
@@ -230,7 +235,8 @@ public final class TablesInitializer {
     DROP("drop"),
     REPAIR("repair"),
     ES_DROP("es-drop"),
-    ES_CREATE("es-create");
+    ES_CREATE("es-create"),
+    ES_MIGRATE("es-migrate");
 
     private final String value;
 

docker/metadata/openmetadata-start.sh

@@ -13,5 +13,5 @@
 while ! wget -O /dev/null -o /dev/null mysql:3306; do sleep 5; done
 cp /openmetadata.yaml /openmetadata-*/conf/openmetadata.yaml
 cd /openmetadata-*/
-./bootstrap/bootstrap_storage.sh drop-create-all
+./bootstrap/bootstrap_storage.sh migrate-all
 ./bin/openmetadata-server-start.sh conf/openmetadata.yaml