diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml index ec2f68b1ecf..2f71591a9ae 100644 --- a/conf/openmetadata.yaml +++ b/conf/openmetadata.yaml @@ -278,7 +278,7 @@ elasticsearch: scheme: ${ELASTICSEARCH_SCHEME:-http} username: ${ELASTICSEARCH_USER:-""} password: ${ELASTICSEARCH_PASSWORD:-""} - clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-""} + clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-"openmetadata"} truststorePath: ${ELASTICSEARCH_TRUST_STORE_PATH:-""} truststorePassword: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""} connectionTimeoutSecs: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} diff --git a/docker/development/docker-compose-postgres.yml b/docker/development/docker-compose-postgres.yml index 214205bbdc1..fc3c6150ca8 100644 --- a/docker/development/docker-compose-postgres.yml +++ b/docker/development/docker-compose-postgres.yml @@ -186,6 +186,7 @@ services: ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""} ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""} SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"} + ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"} ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""} ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""} ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} @@ -400,6 +401,7 @@ services: ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""} ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-"OpenMetadata_password123!!!"} SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"} + ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"} ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""} ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""} ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index a9f6c51504b..6d07bf21eb6 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -185,6 +185,7 @@ services: ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""} ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""} SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"} + ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"} ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""} ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""} ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} @@ -396,6 +397,7 @@ services: ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""} ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""} SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"} + ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"} ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""} ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""} ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} diff --git a/docker/run_local_docker.sh b/docker/run_local_docker.sh index 665428af74a..4bd92e24404 100755 --- a/docker/run_local_docker.sh +++ b/docker/run_local_docker.sh @@ -110,7 +110,7 @@ if [ $RESULT -ne 0 ]; then exit 1 fi -until curl -s -f "http://localhost:9200/_cat/indices/team_search_index"; do +until curl -s -f "http://localhost:9200/_cat/indices/openmetadata_team_search_index"; do echo 'Checking if Elastic Search instance is up...\n' sleep 5 done diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 81008f8d81e..ff888efadff 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.io.InputStream; import java.text.ParseException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -44,6 +45,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeSet; import java.util.UUID; +import java.util.stream.Collectors; import javax.json.JsonObject; import javax.ws.rs.core.Response; import lombok.Getter; @@ -189,9 +191,12 @@ public class SearchRepository { } public String getIndexOrAliasName(String name) { - return clusterAlias != null && !clusterAlias.isEmpty() - ? clusterAlias + indexNameSeparator + name - : name; + if (clusterAlias == null || clusterAlias.isEmpty()) { + return name; + } + return Arrays.stream(name.split(",")) + .map(index -> clusterAlias + indexNameSeparator + index.trim()) + .collect(Collectors.joining(",")); } public String getIndexNameWithoutAlias(String fullIndexName) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 7ab3829c68e..a9c1bb03ef4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -286,6 +286,11 @@ public class ElasticSearchClient implements SearchClient { try { Set aliases = new HashSet<>(indexMapping.getParentAliases(clusterAlias)); aliases.add(indexMapping.getAlias(clusterAlias)); + // Get the child aliases + List childAliases = indexMapping.getChildAliases(clusterAlias); + + // Add the child aliases to the set of aliases + aliases.addAll(childAliases); IndicesAliasesRequest.AliasActions aliasAction = IndicesAliasesRequest.AliasActions.add() .index(indexMapping.getIndexName(clusterAlias)) @@ -1690,7 +1695,8 @@ public class ElasticSearchClient implements SearchClient { Pair fieldAndValue, Pair> updates) { if (isClientAvailable) { - UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); + UpdateByQueryRequest updateByQueryRequest = + new UpdateByQueryRequest(Entity.getSearchRepository().getIndexOrAliasName(indexName)); updateChildren(updateByQueryRequest, fieldAndValue, updates); } } @@ -1740,7 +1746,7 @@ public class ElasticSearchClient implements SearchClient { private void updateElasticSearchByQuery(UpdateByQueryRequest updateByQueryRequest) { if (updateByQueryRequest != null && isClientAvailable) { updateByQueryRequest.setRefresh(true); - LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateByQueryRequest); + LOG.info(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateByQueryRequest); client.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 42af2cf818c..8e3ee055850 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -277,8 +277,13 @@ public class OpenSearchClient implements SearchClient { @Override public void createAliases(IndexMapping indexMapping) { try { - List aliases = indexMapping.getParentAliases(clusterAlias); + Set aliases = new HashSet<>(indexMapping.getParentAliases(clusterAlias)); aliases.add(indexMapping.getAlias(clusterAlias)); + // Get the child aliases + List childAliases = indexMapping.getChildAliases(clusterAlias); + + // Add the child aliases to the set of aliases + aliases.addAll(childAliases); IndicesAliasesRequest.AliasActions aliasAction = IndicesAliasesRequest.AliasActions.add() .index(indexMapping.getIndexName(clusterAlias)) @@ -1673,7 +1678,8 @@ public class OpenSearchClient implements SearchClient { Pair fieldAndValue, Pair> updates) { if (isClientAvailable) { - UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); + UpdateByQueryRequest updateByQueryRequest = + new UpdateByQueryRequest(Entity.getSearchRepository().getIndexOrAliasName(indexName)); updateChildren(updateByQueryRequest, fieldAndValue, updates); } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java index aeccc3b06c2..0445f4beb1c 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java @@ -75,6 +75,7 @@ public abstract class OpenMetadataApplicationTest { public static final Integer ELASTIC_BATCH_SIZE = 10; public static final IndexMappingLanguage ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE = IndexMappingLanguage.EN; + public static final String ELASTIC_SEARCH_CLUSTER_ALIAS = "openmetadata"; public static final ElasticSearchConfiguration.SearchType ELASTIC_SEARCH_TYPE = ElasticSearchConfiguration.SearchType.ELASTICSEARCH; public static DropwizardAppExtension APP; @@ -256,6 +257,7 @@ public abstract class OpenMetadataApplicationTest { .withKeepAliveTimeoutSecs(ELASTIC_KEEP_ALIVE_TIMEOUT) .withBatchSize(ELASTIC_BATCH_SIZE) .withSearchIndexMappingLanguage(ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE) + .withClusterAlias(ELASTIC_SEARCH_CLUSTER_ALIAS) .withSearchType(ELASTIC_SEARCH_TYPE); SearchRepository searchRepository = new SearchRepository(esConfig); LOG.info("creating indexes."); @@ -313,6 +315,8 @@ public abstract class OpenMetadataApplicationTest { ConfigOverride.config( "elasticsearch.searchIndexMappingLanguage", ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE.value())); + configOverrides.add( + ConfigOverride.config("elasticsearch.clusterAlias", ELASTIC_SEARCH_CLUSTER_ALIAS)); configOverrides.add( ConfigOverride.config("elasticsearch.searchType", ELASTIC_SEARCH_TYPE.value())); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index bb1446e4c2c..0d59f662d3b 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -1079,7 +1079,12 @@ public abstract class EntityResourceTest entityIds = new ArrayList<>(); SearchHit[] hits = response.getHits().getHits(); for (SearchHit hit : hits) { @@ -2212,9 +2221,8 @@ public abstract class EntityResourceTest entityIds = new ArrayList<>(); SearchHit[] hits = response.getHits().getHits(); for (SearchHit hit : hits) { @@ -2229,9 +2237,8 @@ public abstract class EntityResourceTest sourceAsMap = hit.getSourceAsMap(); @@ -2321,12 +2328,11 @@ public abstract class EntityResourceTest getEntityDocumentFromSearch(UUID entityId, String entityType) throws HttpResponseException { IndexMapping indexMapping = Entity.getSearchRepository().getIndexMapping(entityType); + // SearchResource.java-searchEntityInEsIndexWithId method internally appends clusterAlias name WebTarget target = getResource( String.format( - "search/get/%s/doc/%s", - indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()), - entityId.toString())); + "search/get/%s/doc/%s", indexMapping.getIndexName(null), entityId.toString())); String result = TestUtils.get(target, String.class, ADMIN_AUTH_HEADERS); GetResponse response = null; try { @@ -3357,7 +3363,12 @@ public abstract class EntityResourceTest { RestClient searchClient = getSearchClient(); IndexMapping index = Entity.getSearchRepository().getIndexMapping(TABLE); Response response; - Request request = new Request("GET", String.format("%s/_search", index.getIndexName(null))); + // Direct request to es needs to have es clusterAlias appended with indexName + Request request = + new Request( + "GET", + String.format( + "%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias()))); String query = "{\"size\": 100,\"query\":{\"bool\":{\"must\":[{\"term\":{\"descriptionStatus\":\"INCOMPLETE\"}}]}}}"; request.setJsonEntity(query); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.tsx index 496fbd64fc8..ae5e381b9dc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.tsx @@ -261,7 +261,7 @@ export const filterOptionsByIndex = ( maxItemsPerType = 5 ) => options - .filter((option) => option._index === searchIndex) + .filter((option) => option._index.includes(searchIndex)) .map((option) => option._source) .slice(0, maxItemsPerType);