Support Elasticsearch cluster alias for E2E tests (#17117)

* Fix backend tests and have index alias with clusterAlias appended

* Fix backend tests and have index alias with clusterAlias appended

* Fix failing tests

* alias setup

* fix suggestion not working due to alias

* fix getIndexOrAliasName method for multiple indexes

* update openmetadata.yaml

* update childAliases with clusterAlias

---------

Co-authored-by: Ashish Gupta <ashish@getcollate.io>
Co-authored-by: Sriharsha Chintalapani <harshach@users.noreply.github.com>

conf/openmetadata.yaml

@@ -278,7 +278,7 @@ elasticsearch:
   scheme: ${ELASTICSEARCH_SCHEME:-http}
   username: ${ELASTICSEARCH_USER:-""}
   password: ${ELASTICSEARCH_PASSWORD:-""}
-  clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-""}
+  clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-"openmetadata"}
   truststorePath: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
   truststorePassword: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
   connectionTimeoutSecs: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}

docker/development/docker-compose-postgres.yml

@@ -186,6 +186,7 @@ services:
       ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
       ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
       SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
+      ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
       ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
       ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
       ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
@@ -400,6 +401,7 @@ services:
       ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
       ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-"OpenMetadata_password123!!!"}
       SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
+      ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
       ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
       ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
       ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}

docker/development/docker-compose.yml

@@ -185,6 +185,7 @@ services:
       ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
       ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
       SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
+      ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
       ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
       ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
       ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
@@ -396,6 +397,7 @@ services:
       ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
       ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
       SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
+      ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
       ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
       ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
       ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}

docker/run_local_docker.sh

@@ -110,7 +110,7 @@ if [ $RESULT -ne 0 ]; then
   exit 1
 fi
 
-until curl -s -f "http://localhost:9200/_cat/indices/team_search_index"; do
+until curl -s -f "http://localhost:9200/_cat/indices/openmetadata_team_search_index"; do
   echo 'Checking if Elastic Search instance is up...\n'
   sleep 5
 done

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java

@@ -33,6 +33,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -44,6 +45,7 @@ import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeSet;
 import java.util.UUID;
+import java.util.stream.Collectors;
 import javax.json.JsonObject;
 import javax.ws.rs.core.Response;
 import lombok.Getter;
@@ -189,9 +191,12 @@ public class SearchRepository {
   }
 
   public String getIndexOrAliasName(String name) {
-    return clusterAlias != null && !clusterAlias.isEmpty()
+    if (clusterAlias == null || clusterAlias.isEmpty()) {
-        ? clusterAlias + indexNameSeparator + name
+      return name;
-        : name;
+    }
+    return Arrays.stream(name.split(","))
+        .map(index -> clusterAlias + indexNameSeparator + index.trim())
+        .collect(Collectors.joining(","));
   }
 
   public String getIndexNameWithoutAlias(String fullIndexName) {

openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java

@@ -286,6 +286,11 @@ public class ElasticSearchClient implements SearchClient {
     try {
       Set<String> aliases = new HashSet<>(indexMapping.getParentAliases(clusterAlias));
       aliases.add(indexMapping.getAlias(clusterAlias));
+      // Get the child aliases
+      List<String> childAliases = indexMapping.getChildAliases(clusterAlias);
+
+      // Add the child aliases to the set of aliases
+      aliases.addAll(childAliases);
       IndicesAliasesRequest.AliasActions aliasAction =
           IndicesAliasesRequest.AliasActions.add()
               .index(indexMapping.getIndexName(clusterAlias))
@@ -1690,7 +1695,8 @@ public class ElasticSearchClient implements SearchClient {
       Pair<String, String> fieldAndValue,
       Pair<String, Map<String, Object>> updates) {
     if (isClientAvailable) {
-      UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName);
+      UpdateByQueryRequest updateByQueryRequest =
+          new UpdateByQueryRequest(Entity.getSearchRepository().getIndexOrAliasName(indexName));
       updateChildren(updateByQueryRequest, fieldAndValue, updates);
     }
   }
@@ -1740,7 +1746,7 @@ public class ElasticSearchClient implements SearchClient {
   private void updateElasticSearchByQuery(UpdateByQueryRequest updateByQueryRequest) {
     if (updateByQueryRequest != null && isClientAvailable) {
       updateByQueryRequest.setRefresh(true);
-      LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateByQueryRequest);
+      LOG.info(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateByQueryRequest);
       client.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT);
     }
   }

openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java

@@ -277,8 +277,13 @@ public class OpenSearchClient implements SearchClient {
   @Override
   public void createAliases(IndexMapping indexMapping) {
     try {
-      List<String> aliases = indexMapping.getParentAliases(clusterAlias);
+      Set<String> aliases = new HashSet<>(indexMapping.getParentAliases(clusterAlias));
       aliases.add(indexMapping.getAlias(clusterAlias));
+      // Get the child aliases
+      List<String> childAliases = indexMapping.getChildAliases(clusterAlias);
+
+      // Add the child aliases to the set of aliases
+      aliases.addAll(childAliases);
       IndicesAliasesRequest.AliasActions aliasAction =
           IndicesAliasesRequest.AliasActions.add()
               .index(indexMapping.getIndexName(clusterAlias))
@@ -1673,7 +1678,8 @@ public class OpenSearchClient implements SearchClient {
       Pair<String, String> fieldAndValue,
       Pair<String, Map<String, Object>> updates) {
     if (isClientAvailable) {
-      UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName);
+      UpdateByQueryRequest updateByQueryRequest =
+          new UpdateByQueryRequest(Entity.getSearchRepository().getIndexOrAliasName(indexName));
       updateChildren(updateByQueryRequest, fieldAndValue, updates);
     }
   }

openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java

@@ -75,6 +75,7 @@ public abstract class OpenMetadataApplicationTest {
   public static final Integer ELASTIC_BATCH_SIZE = 10;
   public static final IndexMappingLanguage ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE =
       IndexMappingLanguage.EN;
+  public static final String ELASTIC_SEARCH_CLUSTER_ALIAS = "openmetadata";
   public static final ElasticSearchConfiguration.SearchType ELASTIC_SEARCH_TYPE =
       ElasticSearchConfiguration.SearchType.ELASTICSEARCH;
   public static DropwizardAppExtension<OpenMetadataApplicationConfig> APP;
@@ -256,6 +257,7 @@ public abstract class OpenMetadataApplicationTest {
         .withKeepAliveTimeoutSecs(ELASTIC_KEEP_ALIVE_TIMEOUT)
         .withBatchSize(ELASTIC_BATCH_SIZE)
         .withSearchIndexMappingLanguage(ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE)
+        .withClusterAlias(ELASTIC_SEARCH_CLUSTER_ALIAS)
         .withSearchType(ELASTIC_SEARCH_TYPE);
     SearchRepository searchRepository = new SearchRepository(esConfig);
     LOG.info("creating indexes.");
@@ -313,6 +315,8 @@ public abstract class OpenMetadataApplicationTest {
         ConfigOverride.config(
             "elasticsearch.searchIndexMappingLanguage",
             ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE.value()));
+    configOverrides.add(
+        ConfigOverride.config("elasticsearch.clusterAlias", ELASTIC_SEARCH_CLUSTER_ALIAS));
     configOverrides.add(
         ConfigOverride.config("elasticsearch.searchType", ELASTIC_SEARCH_TYPE.value()));
   }

openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java

@@ -1079,7 +1079,12 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
     RestClient searchClient = getSearchClient();
     IndexMapping index = Entity.getSearchRepository().getIndexMapping(entityType);
     Response response;
-    Request request = new Request("GET", String.format("%s/_search", index.getIndexName(null)));
+    // Direct request to es needs to have es clusterAlias appended with indexName
+    Request request =
+        new Request(
+            "GET",
+            String.format(
+                "%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
     String query =
         "{\"size\": 100,\"query\":{\"bool\":{\"must\":[{\"term\":{\"descriptionStatus\":\"INCOMPLETE\"}}]}}}";
     request.setJsonEntity(query);
@@ -1124,7 +1129,12 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
     RestClient searchClient = getSearchClient();
     IndexMapping index = Entity.getSearchRepository().getIndexMapping(entityType);
     Response response;
-    Request request = new Request("GET", String.format("%s/_search", index.getIndexName(null)));
+    // Direct request to es needs to have es clusterAlias appended with indexName
+    Request request =
+        new Request(
+            "GET",
+            String.format(
+                "%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
     String query =
         "{\"size\": 100,\"query\":{\"bool\":{\"must\":[{\"term\":{\"descriptionStatus\":\"INCOMPLETE\"}}]}}}";
     request.setJsonEntity(query);
@@ -2190,9 +2200,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
     EntityReference entityReference = getEntityReference(entity);
     IndexMapping indexMapping =
         Entity.getSearchRepository().getIndexMapping(entityReference.getType());
-    SearchResponse response =
+    // search api method internally appends clusterAlias name
-        getResponseFormSearch(
+    SearchResponse response = getResponseFormSearch(indexMapping.getIndexName(null));
-            indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
     List<String> entityIds = new ArrayList<>();
     SearchHit[] hits = response.getHits().getHits();
     for (SearchHit hit : hits) {
@@ -2212,9 +2221,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
     EntityReference entityReference = getEntityReference(entity);
     IndexMapping indexMapping =
         Entity.getSearchRepository().getIndexMapping(entityReference.getType());
-    SearchResponse response =
+    // search api method internally appends clusterAlias name
-        getResponseFormSearch(
+    SearchResponse response = getResponseFormSearch(indexMapping.getIndexName(null));
-            indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
     List<String> entityIds = new ArrayList<>();
     SearchHit[] hits = response.getHits().getHits();
     for (SearchHit hit : hits) {
@@ -2229,9 +2237,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
     TestUtils.delete(target, entityClass, ADMIN_AUTH_HEADERS);
     // search again in search after deleting
 
-    response =
+    // search api method internally appends clusterAlias name
-        getResponseFormSearch(
+    response = getResponseFormSearch(indexMapping.getIndexName(null));
-            indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
     hits = response.getHits().getHits();
     for (SearchHit hit : hits) {
       Map<String, Object> sourceAsMap = hit.getSourceAsMap();
@@ -2321,12 +2328,11 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
   public static Map<String, Object> getEntityDocumentFromSearch(UUID entityId, String entityType)
       throws HttpResponseException {
     IndexMapping indexMapping = Entity.getSearchRepository().getIndexMapping(entityType);
+    // SearchResource.java-searchEntityInEsIndexWithId method internally appends clusterAlias name
     WebTarget target =
         getResource(
             String.format(
-                "search/get/%s/doc/%s",
+                "search/get/%s/doc/%s", indexMapping.getIndexName(null), entityId.toString()));
-                indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()),
-                entityId.toString()));
     String result = TestUtils.get(target, String.class, ADMIN_AUTH_HEADERS);
     GetResponse response = null;
     try {
@@ -3357,7 +3363,12 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
       throws IOException {
     RestClient searchClient = getSearchClient();
     IndexMapping index = Entity.getSearchRepository().getIndexMapping(entityType);
-    Request request = new Request("GET", String.format("%s/_search", index.getIndexName(null)));
+    // Direct request to es needs to have es clusterAlias appended with indexName
+    Request request =
+        new Request(
+            "GET",
+            String.format(
+                "%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
     String query =
         String.format(
             "{\"query\":{\"bool\":{\"filter\":[{\"term\":{\"_id\":\"%s\"}}]}}}", entity.getId());

openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/ReportDataResourceTest.java

@@ -25,17 +25,20 @@ import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import javax.ws.rs.client.WebTarget;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.http.client.HttpResponseException;
 import org.junit.jupiter.api.Test;
 import org.openmetadata.schema.analytics.EntityReportData;
 import org.openmetadata.schema.analytics.ReportData;
 import org.openmetadata.schema.analytics.WebAnalyticUserActivityReportData;
 import org.openmetadata.schema.type.MetadataOperation;
+import org.openmetadata.service.Entity;
 import org.openmetadata.service.OpenMetadataApplicationTest;
 import org.openmetadata.service.resources.analytics.ReportDataResource.ReportDataResultList;
 import org.openmetadata.service.util.ResultList;
 import org.openmetadata.service.util.TestUtils;
 
+@Slf4j
 class ReportDataResourceTest extends OpenMetadataApplicationTest {
 
   public static final String JSON_QUERY =
@@ -228,7 +231,12 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest {
   private JsonNode runSearchQuery(String query, String index) throws IOException {
     RestClient searchClient = getSearchClient();
     Response response;
-    Request request = new Request("POST", String.format("/%s/_search", index));
+    Request request =
+        new Request(
+            "POST",
+            String.format(
+                "/%s/_search",
+                Entity.getSearchRepository().getIndexOrAliasName(String.valueOf(index))));
     request.setJsonEntity(query);
     try {
       response = searchClient.performRequest(request);

openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java

@@ -2410,7 +2410,12 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
     RestClient searchClient = getSearchClient();
     IndexMapping index = Entity.getSearchRepository().getIndexMapping(TABLE);
     Response response;
-    Request request = new Request("GET", String.format("%s/_search", index.getIndexName(null)));
+    // Direct request to es needs to have es clusterAlias appended with indexName
+    Request request =
+        new Request(
+            "GET",
+            String.format(
+                "%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
     String query =
         "{\"size\": 100,\"query\":{\"bool\":{\"must\":[{\"term\":{\"descriptionStatus\":\"INCOMPLETE\"}}]}}}";
     request.setJsonEntity(query);

openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.tsx

@@ -261,7 +261,7 @@ export const filterOptionsByIndex = (
   maxItemsPerType = 5
 ) =>
   options
-    .filter((option) => option._index === searchIndex)
+    .filter((option) => option._index.includes(searchIndex))
     .map((option) => option._source)
     .slice(0, maxItemsPerType);