yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-03 12:08:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support Elasticsearch cluster alias for E2E tests  (#17117)

Browse Source 
* Fix backend tests and have index alias with clusterAlias appended

* Fix backend tests and have index alias with clusterAlias appended

* Fix failing tests

* alias setup

* fix suggestion not working due to alias

* fix getIndexOrAliasName method for multiple indexes

* update openmetadata.yaml

* update childAliases with clusterAlias

---------

Co-authored-by: Ashish Gupta <ashish@getcollate.io>
Co-authored-by: Sriharsha Chintalapani <harshach@users.noreply.github.com>
This commit is contained in:
sonika-shah 2024-08-10 23:56:49 +05:30 committed by GitHub
parent ab1e6b7acd
commit 8d5661a7bf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 76 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
conf/openmetadata.yaml
View File

@ -278,7 +278,7 @@ elasticsearch:
scheme: ${ELASTICSEARCH_SCHEME:-http}
username: ${ELASTICSEARCH_USER:-""}
password: ${ELASTICSEARCH_PASSWORD:-""}
clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-""}
clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-"openmetadata"}
truststorePath: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
truststorePassword: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
connectionTimeoutSecs: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}

2
docker/development/docker-compose-postgres.yml
View File

@ -186,6 +186,7 @@ services:
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
@ -400,6 +401,7 @@ services:
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-"OpenMetadata_password123!!!"}
SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}

2
docker/development/docker-compose.yml
View File

@ -185,6 +185,7 @@ services:
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
@ -396,6 +397,7 @@ services:
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}

2
docker/run_local_docker.sh
View File

@ -110,7 +110,7 @@ if [ $RESULT -ne 0 ]; then
exit 1
fi
until curl -s -f "http://localhost:9200/_cat/indices/team_search_index"; do
until curl -s -f "http://localhost:9200/_cat/indices/openmetadata_team_search_index"; do
echo 'Checking if Elastic Search instance is up...\n'
sleep 5
done

11
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java
View File

@ -33,6 +33,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
@ -44,6 +45,7 @@ import java.util.Set;
import java.util.SortedMap;
import java.util.TreeSet;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.json.JsonObject;
import javax.ws.rs.core.Response;
import lombok.Getter;
@ -189,9 +191,12 @@ public class SearchRepository {
}
public String getIndexOrAliasName(String name) {
return clusterAlias != null && !clusterAlias.isEmpty()
? clusterAlias + indexNameSeparator + name
: name;
if (clusterAlias == null || clusterAlias.isEmpty()) {
return name;
}
return Arrays.stream(name.split(","))
.map(index -> clusterAlias + indexNameSeparator + index.trim())
.collect(Collectors.joining(","));
}
public String getIndexNameWithoutAlias(String fullIndexName) {

10
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -286,6 +286,11 @@ public class ElasticSearchClient implements SearchClient {
try {
Set<String> aliases = new HashSet<>(indexMapping.getParentAliases(clusterAlias));
aliases.add(indexMapping.getAlias(clusterAlias));
// Get the child aliases
List<String> childAliases = indexMapping.getChildAliases(clusterAlias);
// Add the child aliases to the set of aliases
aliases.addAll(childAliases);
IndicesAliasesRequest.AliasActions aliasAction =
IndicesAliasesRequest.AliasActions.add()
.index(indexMapping.getIndexName(clusterAlias))
@ -1690,7 +1695,8 @@ public class ElasticSearchClient implements SearchClient {
Pair<String, String> fieldAndValue,
Pair<String, Map<String, Object>> updates) {
if (isClientAvailable) {
UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName);
UpdateByQueryRequest updateByQueryRequest =
new UpdateByQueryRequest(Entity.getSearchRepository().getIndexOrAliasName(indexName));
updateChildren(updateByQueryRequest, fieldAndValue, updates);
}
}
@ -1740,7 +1746,7 @@ public class ElasticSearchClient implements SearchClient {
private void updateElasticSearchByQuery(UpdateByQueryRequest updateByQueryRequest) {
if (updateByQueryRequest != null && isClientAvailable) {
updateByQueryRequest.setRefresh(true);
LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateByQueryRequest);
LOG.info(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateByQueryRequest);
client.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT);
}
}

10
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -277,8 +277,13 @@ public class OpenSearchClient implements SearchClient {
@Override
public void createAliases(IndexMapping indexMapping) {
try {
List<String> aliases = indexMapping.getParentAliases(clusterAlias);
Set<String> aliases = new HashSet<>(indexMapping.getParentAliases(clusterAlias));
aliases.add(indexMapping.getAlias(clusterAlias));
// Get the child aliases
List<String> childAliases = indexMapping.getChildAliases(clusterAlias);
// Add the child aliases to the set of aliases
aliases.addAll(childAliases);
IndicesAliasesRequest.AliasActions aliasAction =
IndicesAliasesRequest.AliasActions.add()
.index(indexMapping.getIndexName(clusterAlias))
@ -1673,7 +1678,8 @@ public class OpenSearchClient implements SearchClient {
Pair<String, String> fieldAndValue,
Pair<String, Map<String, Object>> updates) {
if (isClientAvailable) {
UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName);
UpdateByQueryRequest updateByQueryRequest =
new UpdateByQueryRequest(Entity.getSearchRepository().getIndexOrAliasName(indexName));
updateChildren(updateByQueryRequest, fieldAndValue, updates);
}
}

4
openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java
View File

@ -75,6 +75,7 @@ public abstract class OpenMetadataApplicationTest {
public static final Integer ELASTIC_BATCH_SIZE = 10;
public static final IndexMappingLanguage ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE =
IndexMappingLanguage.EN;
public static final String ELASTIC_SEARCH_CLUSTER_ALIAS = "openmetadata";
public static final ElasticSearchConfiguration.SearchType ELASTIC_SEARCH_TYPE =
ElasticSearchConfiguration.SearchType.ELASTICSEARCH;
public static DropwizardAppExtension<OpenMetadataApplicationConfig> APP;
@ -256,6 +257,7 @@ public abstract class OpenMetadataApplicationTest {
.withKeepAliveTimeoutSecs(ELASTIC_KEEP_ALIVE_TIMEOUT)
.withBatchSize(ELASTIC_BATCH_SIZE)
.withSearchIndexMappingLanguage(ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE)
.withClusterAlias(ELASTIC_SEARCH_CLUSTER_ALIAS)
.withSearchType(ELASTIC_SEARCH_TYPE);
SearchRepository searchRepository = new SearchRepository(esConfig);
LOG.info("creating indexes.");
@ -313,6 +315,8 @@ public abstract class OpenMetadataApplicationTest {
ConfigOverride.config(
"elasticsearch.searchIndexMappingLanguage",
ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE.value()));
configOverrides.add(
ConfigOverride.config("elasticsearch.clusterAlias", ELASTIC_SEARCH_CLUSTER_ALIAS));
configOverrides.add(
ConfigOverride.config("elasticsearch.searchType", ELASTIC_SEARCH_TYPE.value()));
}

41
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -1079,7 +1079,12 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
RestClient searchClient = getSearchClient();
IndexMapping index = Entity.getSearchRepository().getIndexMapping(entityType);
Response response;
Request request = new Request("GET", String.format("%s/_search", index.getIndexName(null)));
// Direct request to es needs to have es clusterAlias appended with indexName
Request request =
new Request(
"GET",
String.format(
"%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
String query =
"{\"size\": 100,\"query\":{\"bool\":{\"must\":[{\"term\":{\"descriptionStatus\":\"INCOMPLETE\"}}]}}}";
request.setJsonEntity(query);
@ -1124,7 +1129,12 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
RestClient searchClient = getSearchClient();
IndexMapping index = Entity.getSearchRepository().getIndexMapping(entityType);
Response response;
Request request = new Request("GET", String.format("%s/_search", index.getIndexName(null)));
// Direct request to es needs to have es clusterAlias appended with indexName
Request request =
new Request(
"GET",
String.format(
"%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
String query =
"{\"size\": 100,\"query\":{\"bool\":{\"must\":[{\"term\":{\"descriptionStatus\":\"INCOMPLETE\"}}]}}}";
request.setJsonEntity(query);
@ -2190,9 +2200,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
EntityReference entityReference = getEntityReference(entity);
IndexMapping indexMapping =
Entity.getSearchRepository().getIndexMapping(entityReference.getType());
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
// search api method internally appends clusterAlias name
SearchResponse response = getResponseFormSearch(indexMapping.getIndexName(null));
List<String> entityIds = new ArrayList<>();
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
@ -2212,9 +2221,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
EntityReference entityReference = getEntityReference(entity);
IndexMapping indexMapping =
Entity.getSearchRepository().getIndexMapping(entityReference.getType());
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
// search api method internally appends clusterAlias name
SearchResponse response = getResponseFormSearch(indexMapping.getIndexName(null));
List<String> entityIds = new ArrayList<>();
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
@ -2229,9 +2237,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
TestUtils.delete(target, entityClass, ADMIN_AUTH_HEADERS);
// search again in search after deleting
response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
// search api method internally appends clusterAlias name
response = getResponseFormSearch(indexMapping.getIndexName(null));
hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
@ -2321,12 +2328,11 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
public static Map<String, Object> getEntityDocumentFromSearch(UUID entityId, String entityType)
throws HttpResponseException {
IndexMapping indexMapping = Entity.getSearchRepository().getIndexMapping(entityType);
// SearchResource.java-searchEntityInEsIndexWithId method internally appends clusterAlias name
WebTarget target =
getResource(
String.format(
"search/get/%s/doc/%s",
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()),
entityId.toString()));
"search/get/%s/doc/%s", indexMapping.getIndexName(null), entityId.toString()));
String result = TestUtils.get(target, String.class, ADMIN_AUTH_HEADERS);
GetResponse response = null;
try {
@ -3357,7 +3363,12 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
throws IOException {
RestClient searchClient = getSearchClient();
IndexMapping index = Entity.getSearchRepository().getIndexMapping(entityType);
Request request = new Request("GET", String.format("%s/_search", index.getIndexName(null)));
// Direct request to es needs to have es clusterAlias appended with indexName
Request request =
new Request(
"GET",
String.format(
"%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
String query =
String.format(
"{\"query\":{\"bool\":{\"filter\":[{\"term\":{\"_id\":\"%s\"}}]}}}", entity.getId());

10
openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/ReportDataResourceTest.java
View File

@ -25,17 +25,20 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.ws.rs.client.WebTarget;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.Test;
import org.openmetadata.schema.analytics.EntityReportData;
import org.openmetadata.schema.analytics.ReportData;
import org.openmetadata.schema.analytics.WebAnalyticUserActivityReportData;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationTest;
import org.openmetadata.service.resources.analytics.ReportDataResource.ReportDataResultList;
import org.openmetadata.service.util.ResultList;
import org.openmetadata.service.util.TestUtils;
@Slf4j
class ReportDataResourceTest extends OpenMetadataApplicationTest {
public static final String JSON_QUERY =
@ -228,7 +231,12 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest {
private JsonNode runSearchQuery(String query, String index) throws IOException {
RestClient searchClient = getSearchClient();
Response response;
Request request = new Request("POST", String.format("/%s/_search", index));
Request request =
new Request(
"POST",
String.format(
"/%s/_search",
Entity.getSearchRepository().getIndexOrAliasName(String.valueOf(index))));
request.setJsonEntity(query);
try {
response = searchClient.performRequest(request);

7
openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java
View File

@ -2410,7 +2410,12 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
RestClient searchClient = getSearchClient();
IndexMapping index = Entity.getSearchRepository().getIndexMapping(TABLE);
Response response;
Request request = new Request("GET", String.format("%s/_search", index.getIndexName(null)));
// Direct request to es needs to have es clusterAlias appended with indexName
Request request =
new Request(
"GET",
String.format(
"%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
String query =
"{\"size\": 100,\"query\":{\"bool\":{\"must\":[{\"term\":{\"descriptionStatus\":\"INCOMPLETE\"}}]}}}";
request.setJsonEntity(query);

2
openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.tsx
View File

@ -261,7 +261,7 @@ export const filterOptionsByIndex = (
maxItemsPerType = 5
) =>
options
.filter((option) => option._index === searchIndex)
.filter((option) => option._index.includes(searchIndex))
.map((option) => option._source)
.slice(0, maxItemsPerType);