yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-18 05:57:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #1814: Add migrate-all option to update ElasticSearch index mappings (#1815)

Browse Source 
* Fix #1814: Add migrate-all option to update ElasticSearch index mappings

* Fix #1814: Add migrate-all option to update ElasticSearch index mappings
This commit is contained in:
Sriharsha Chintalapani 2021-12-16 20:56:58 -08:00 committed by GitHub
parent df6232ef52
commit 39e4163d6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 48 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bootstrap/bootstrap_storage.sh
View File

@ -88,6 +88,9 @@ drop-create )
drop-create-all )
execute "drop" && execute "create" && execute "es-drop" && execute "es-create"
;;
migrate-all )
execute "migrate" && execute "es-migrate"
;;
*)
printUsage
exit 1

37
catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java
View File

@ -27,6 +27,7 @@ import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.client.indices.PutMappingRequest;
import org.elasticsearch.common.xcontent.XContentType;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Dashboard;
@ -84,6 +85,16 @@ public class ElasticSearchIndexDefinition {
}
}
public void updateIndexes() {
try {
for (ElasticSearchIndexType elasticSearchIndexType : ElasticSearchIndexType.values()) {
updateIndex(elasticSearchIndexType);
}
} catch (Exception e) {
LOG.error("Failed to created Elastic Search indexes due to", e);
}
}
public void dropIndexes() {
try {
for (ElasticSearchIndexType elasticSearchIndexType : ElasticSearchIndexType.values()) {
@ -123,6 +134,32 @@ public class ElasticSearchIndexDefinition {
return true;
}
private boolean updateIndex(ElasticSearchIndexType elasticSearchIndexType) {
try {
GetIndexRequest gRequest = new GetIndexRequest(elasticSearchIndexType.indexName);
gRequest.local(false);
boolean exists = client.indices().exists(gRequest, RequestOptions.DEFAULT);
String elasticSearchIndexMapping = getIndexMapping(elasticSearchIndexType);
if (exists) {
PutMappingRequest request = new PutMappingRequest(elasticSearchIndexType.indexName);
request.source(elasticSearchIndexMapping, XContentType.JSON);
AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
LOG.info(elasticSearchIndexType.indexName + " Updated " + putMappingResponse.isAcknowledged());
} else {
CreateIndexRequest request = new CreateIndexRequest(elasticSearchIndexType.indexName);
request.mapping(elasticSearchIndexMapping, XContentType.JSON);
CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT);
LOG.info(elasticSearchIndexType.indexName + " Created " + createIndexResponse.isAcknowledged());
}
setIndexStatus(elasticSearchIndexType, ElasticSearchIndexStatus.CREATED);
} catch (Exception e) {
setIndexStatus(elasticSearchIndexType, ElasticSearchIndexStatus.FAILED);
LOG.error("Failed to created Elastic Search indexes due to", e);
return false;
}
return true;
}
private boolean deleteIndex(ElasticSearchIndexType elasticSearchIndexType) {
try {
DeleteIndexRequest request = new DeleteIndexRequest(elasticSearchIndexType.indexName);

8
catalog-rest-service/src/main/java/org/openmetadata/catalog/util/TablesInitializer.java
View File

@ -76,6 +76,7 @@ public final class TablesInitializer {
null, SchemaMigrationOption.ES_CREATE.toString(), false, "Creates all the indexes in the elastic search");
OPTIONS.addOption(
null, SchemaMigrationOption.ES_DROP.toString(), false, "Drop all the indexes in the elastic search");
OPTIONS.addOption(null, SchemaMigrationOption.ES_MIGRATE.toString(), false, "Update Elastic Search index mapping");
}
private TablesInitializer() {}
@ -207,6 +208,10 @@ public final class TablesInitializer {
esIndexDefinition = new ElasticSearchIndexDefinition(client);
esIndexDefinition.createIndexes();
break;
case ES_MIGRATE:
esIndexDefinition = new ElasticSearchIndexDefinition(client);
esIndexDefinition.updateIndexes();
break;
case ES_DROP:
esIndexDefinition = new ElasticSearchIndexDefinition(client);
esIndexDefinition.dropIndexes();
@ -230,7 +235,8 @@ public final class TablesInitializer {
DROP("drop"),
REPAIR("repair"),
ES_DROP("es-drop"),
ES_CREATE("es-create");
ES_CREATE("es-create"),
ES_MIGRATE("es-migrate");
private final String value;

2
docker/metadata/openmetadata-start.sh
View File

@ -13,5 +13,5 @@
while ! wget -O /dev/null -o /dev/null mysql:3306; do sleep 5; done
cp /openmetadata.yaml /openmetadata-*/conf/openmetadata.yaml
cd /openmetadata-*/
./bootstrap/bootstrap_storage.sh drop-create-all
./bootstrap/bootstrap_storage.sh migrate-all
./bin/openmetadata-server-start.sh conf/openmetadata.yaml