yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-03 06:03:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

[MINOR] Enable ES Tests for Maven CI (#15657)

Browse Source 
* fix: added ES test by default

* style: ran java linting

* fix: increase wait time + make es container not reusable

* fix: updated query search index + increase startup and wait time

* fix: Change order of attribute setting for ES container

* style: ran java linting

* fix: order test and increase wait time for ES in Report test

* fix: adjusted flaky es test
This commit is contained in:
Teddy 2024-03-24 19:35:48 +01:00 committed by GitHub
parent 9329b5a488
commit b9bc8a1fa1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 423 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
View File

@ -43,7 +43,7 @@ public interface SearchClient {
"if((ctx._source.%s == null) || (ctx._source.%s.id == '%s')) { ctx._source.put('%s', params)}";
String SOFT_DELETE_RESTORE_SCRIPT = "ctx._source.put('deleted', '%s')";
String REMOVE_TAGS_CHILDREN_SCRIPT =
"for (int i = 0; i < ctx._source.tags.length; i++) { if (ctx._source.tags[i].tagFQN == '%s') { ctx._source.tags.remove(i) }}";
"for (int i = 0; i < ctx._source.tags.length; i++) { if (ctx._source.tags[i].tagFQN == params.fqn) { ctx._source.tags.remove(i) }}";
String REMOVE_LINEAGE_SCRIPT =
"for (int i = 0; i < ctx._source.lineage.length; i++) { if (ctx._source.lineage[i].doc_id == '%s') { ctx._source.lineage.remove(i) }}";

5
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java
View File

@ -28,6 +28,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@ -532,7 +533,9 @@ public class SearchRepository {
case Entity.TAG, Entity.GLOSSARY_TERM -> searchClient.updateChildren(
GLOBAL_SEARCH_ALIAS,
new ImmutablePair<>("tags.tagFQN", entity.getFullyQualifiedName()),
new ImmutablePair<>(REMOVE_TAGS_CHILDREN_SCRIPT, null));
new ImmutablePair<>(
REMOVE_TAGS_CHILDREN_SCRIPT,
Collections.singletonMap("fqn", entity.getFullyQualifiedName())));
case Entity.TEST_SUITE -> {
TestSuite testSuite = (TestSuite) entity;
if (Boolean.TRUE.equals(testSuite.getExecutable())) {

53
openmetadata-service/src/main/resources/elasticsearch/en/query_index_mapping.json
View File

@ -302,6 +302,59 @@
"service_suggest": {
"type": "completion"
},
"tier": {
"properties": {
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"labelType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"source": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"state": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tagFQN": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
}
}
},
"totalVotes": {
"type": "long"
},

53
openmetadata-service/src/main/resources/elasticsearch/jp/query_index_mapping.json
View File

@ -337,6 +337,59 @@
},
"service_suggest": {
"type": "completion"
},
"tier": {
"properties": {
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"labelType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"source": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"state": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tagFQN": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
}
}
}
}
}

53
openmetadata-service/src/main/resources/elasticsearch/zh/query_index_mapping.json
View File

@ -343,6 +343,59 @@
},
"service_suggest": {
"type": "completion"
},
"tier": {
"properties": {
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"labelType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"source": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"state": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tagFQN": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
}
}
}
}
}

107
openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java
View File

@ -17,11 +17,13 @@ import static java.lang.String.format;
import static org.openmetadata.service.util.TablesInitializer.validateAndRunSystemDataMigrations;
import es.org.elasticsearch.client.RestClient;
import es.org.elasticsearch.client.RestClientBuilder;
import io.dropwizard.jersey.jackson.JacksonFeature;
import io.dropwizard.testing.ConfigOverride;
import io.dropwizard.testing.ResourceHelpers;
import io.dropwizard.testing.junit5.DropwizardAppExtension;
import java.net.URI;
import java.time.Duration;
import java.util.HashSet;
import java.util.Set;
import javax.ws.rs.client.Client;
@ -29,6 +31,10 @@ import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.WebTarget;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.flywaydb.core.Flyway;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.client.ClientProperties;
@ -41,12 +47,16 @@ import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.TestInstance;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
import org.openmetadata.schema.type.IndexMappingLanguage;
import org.openmetadata.service.fernet.Fernet;
import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator;
import org.openmetadata.service.jdbi3.locator.ConnectionType;
import org.openmetadata.service.resources.CollectionRegistry;
import org.openmetadata.service.resources.events.WebhookCallbackResource;
import org.openmetadata.service.search.SearchRepository;
import org.testcontainers.containers.JdbcDatabaseContainer;
import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
@Slf4j
@ -54,6 +64,17 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer;
public abstract class OpenMetadataApplicationTest {
protected static final String CONFIG_PATH =
ResourceHelpers.resourceFilePath("openmetadata-secure-test.yaml");
public static final String ELASTIC_USER = "elastic";
public static final String ELASTIC_PASSWORD = "password";
public static final String ELASTIC_SCHEME = "http";
public static final Integer ELASTIC_CONNECT_TIMEOUT = 5;
public static final Integer ELASTIC_SOCKET_TIMEOUT = 60;
public static final Integer ELASTIC_KEEP_ALIVE_TIMEOUT = 600;
public static final Integer ELASTIC_BATCH_SIZE = 10;
public static final IndexMappingLanguage ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE =
IndexMappingLanguage.EN;
public static final ElasticSearchConfiguration.SearchType ELASTIC_SEARCH_TYPE =
ElasticSearchConfiguration.SearchType.ELASTICSEARCH;
public static DropwizardAppExtension<OpenMetadataApplicationConfig> APP;
protected static final WebhookCallbackResource webhookCallbackResource =
new WebhookCallbackResource();
@ -61,15 +82,13 @@ public abstract class OpenMetadataApplicationTest {
public static Jdbi jdbi;
private static ElasticsearchContainer ELASTIC_SEARCH_CONTAINER;
public static final boolean RUN_ELASTIC_SEARCH_TESTCASES = false;
protected static final Set<ConfigOverride> configOverrides = new HashSet<>();
private static final String JDBC_CONTAINER_CLASS_NAME =
"org.testcontainers.containers.MySQLContainer";
private static final String JDBC_CONTAINER_IMAGE = "mysql:8";
private static final String ELASTIC_SEARCH_CONTAINER_IMAGE =
"docker.elastic.co/elasticsearch/elasticsearch:7.16.3";
"docker.elastic.co/elasticsearch/elasticsearch:8.10.2";
private static String HOST;
private static String PORT;
@ -140,14 +159,19 @@ public abstract class OpenMetadataApplicationTest {
flyway.migrate();
ELASTIC_SEARCH_CONTAINER = new ElasticsearchContainer(elasticSearchContainerImage);
if (RUN_ELASTIC_SEARCH_TESTCASES) {
ELASTIC_SEARCH_CONTAINER.start();
ELASTIC_SEARCH_CONTAINER.withReuse(true);
String[] parts = ELASTIC_SEARCH_CONTAINER.getHttpHostAddress().split(":");
HOST = parts[0];
PORT = parts[1];
overrideElasticSearchConfig();
}
ELASTIC_SEARCH_CONTAINER.withPassword("password");
ELASTIC_SEARCH_CONTAINER.withEnv("discovery.type", "single-node");
ELASTIC_SEARCH_CONTAINER.withEnv("xpack.security.enabled", "false");
ELASTIC_SEARCH_CONTAINER.withReuse(false);
ELASTIC_SEARCH_CONTAINER.setWaitStrategy(
new LogMessageWaitStrategy()
.withRegEx(".*(\"message\":\\s?\"started[\\s?|\"].*|] started\n$)")
.withStartupTimeout(Duration.ofMinutes(5)));
ELASTIC_SEARCH_CONTAINER.start();
String[] parts = ELASTIC_SEARCH_CONTAINER.getHttpHostAddress().split(":");
HOST = parts[0];
PORT = parts[1];
overrideElasticSearchConfig();
overrideDatabaseConfig(sqlContainer);
// Migration overrides
@ -172,6 +196,7 @@ public abstract class OpenMetadataApplicationTest {
nativeMigrationScriptsLocation,
extensionMigrationScripsLocation,
false);
createIndices();
APP.before();
createClient();
}
@ -208,9 +233,36 @@ public abstract class OpenMetadataApplicationTest {
}
}
private void createIndices() {
ElasticSearchConfiguration esConfig = new ElasticSearchConfiguration();
esConfig
.withHost(HOST)
.withPort(ELASTIC_SEARCH_CONTAINER.getMappedPort(9200))
.withUsername(ELASTIC_USER)
.withPassword(ELASTIC_PASSWORD)
.withScheme(ELASTIC_SCHEME)
.withConnectionTimeoutSecs(ELASTIC_CONNECT_TIMEOUT)
.withSocketTimeoutSecs(ELASTIC_SOCKET_TIMEOUT)
.withKeepAliveTimeoutSecs(ELASTIC_KEEP_ALIVE_TIMEOUT)
.withBatchSize(ELASTIC_BATCH_SIZE)
.withSearchIndexMappingLanguage(ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE)
.withSearchType(ELASTIC_SEARCH_TYPE);
SearchRepository searchRepository = new SearchRepository(esConfig);
LOG.info("creating indexes.");
searchRepository.createIndexes();
}
public static RestClient getSearchClient() {
return RestClient.builder(HttpHost.create(ELASTIC_SEARCH_CONTAINER.getHttpHostAddress()))
.build();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(ELASTIC_USER, "password"));
RestClientBuilder builder =
RestClient.builder(HttpHost.create(ELASTIC_SEARCH_CONTAINER.getHttpHostAddress()))
.setHttpClientConfigCallback(
httpClientBuilder ->
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
return builder.build();
}
public static WebTarget getResource(String collection) {
@ -231,17 +283,28 @@ public abstract class OpenMetadataApplicationTest {
// elastic search overrides
configOverrides.add(ConfigOverride.config("elasticsearch.host", HOST));
configOverrides.add(ConfigOverride.config("elasticsearch.port", PORT));
configOverrides.add(ConfigOverride.config("elasticsearch.scheme", "http"));
configOverrides.add(ConfigOverride.config("elasticsearch.username", ""));
configOverrides.add(ConfigOverride.config("elasticsearch.password", ""));
configOverrides.add(ConfigOverride.config("elasticsearch.scheme", ELASTIC_SCHEME));
configOverrides.add(ConfigOverride.config("elasticsearch.username", ELASTIC_USER));
configOverrides.add(ConfigOverride.config("elasticsearch.password", ELASTIC_PASSWORD));
configOverrides.add(ConfigOverride.config("elasticsearch.truststorePath", ""));
configOverrides.add(ConfigOverride.config("elasticsearch.truststorePassword", ""));
configOverrides.add(ConfigOverride.config("elasticsearch.connectionTimeoutSecs", "5"));
configOverrides.add(ConfigOverride.config("elasticsearch.socketTimeoutSecs", "60"));
configOverrides.add(ConfigOverride.config("elasticsearch.keepAliveTimeoutSecs", "600"));
configOverrides.add(ConfigOverride.config("elasticsearch.batchSize", "10"));
configOverrides.add(ConfigOverride.config("elasticsearch.searchIndexMappingLanguage", "EN"));
configOverrides.add(ConfigOverride.config("elasticsearch.searchType", "elasticsearch"));
configOverrides.add(
ConfigOverride.config(
"elasticsearch.connectionTimeoutSecs", ELASTIC_CONNECT_TIMEOUT.toString()));
configOverrides.add(
ConfigOverride.config(
"elasticsearch.socketTimeoutSecs", ELASTIC_SOCKET_TIMEOUT.toString()));
configOverrides.add(
ConfigOverride.config(
"elasticsearch.keepAliveTimeoutSecs", ELASTIC_KEEP_ALIVE_TIMEOUT.toString()));
configOverrides.add(
ConfigOverride.config("elasticsearch.batchSize", ELASTIC_BATCH_SIZE.toString()));
configOverrides.add(
ConfigOverride.config(
"elasticsearch.searchIndexMappingLanguage",
ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE.value()));
configOverrides.add(
ConfigOverride.config("elasticsearch.searchType", ELASTIC_SEARCH_TYPE.value()));
}
private static void overrideDatabaseConfig(JdbcDatabaseContainer<?> sqlContainer) {

272
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -1927,170 +1927,163 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
@Test
protected void checkIndexCreated() throws IOException, JSONException {
if (RUN_ELASTIC_SEARCH_TESTCASES) {
RestClient client = getSearchClient();
Request request = new Request("GET", "/_cat/indices");
request.addParameter("format", "json");
Response response = client.performRequest(request);
JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity()));
List<String> indexNamesFromResponse = new ArrayList<>();
for (int i = 0; i < jsonArray.length(); i++) {
JSONObject jsonObject = jsonArray.getJSONObject(i);
String indexName = jsonObject.getString("index");
indexNamesFromResponse.add(indexName);
}
client.close();
RestClient client = getSearchClient();
Request request = new Request("GET", "/_cat/indices");
request.addParameter("format", "json");
Response response = client.performRequest(request);
JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity()));
List<String> indexNamesFromResponse = new ArrayList<>();
for (int i = 0; i < jsonArray.length(); i++) {
JSONObject jsonObject = jsonArray.getJSONObject(i);
String indexName = jsonObject.getString("index");
indexNamesFromResponse.add(indexName);
}
client.close();
}
@Test
protected void checkCreatedEntity(TestInfo test) throws IOException, InterruptedException {
if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
// create entity
T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS);
EntityReference entityReference = getEntityReference(entity);
IndexMapping indexMapping =
Entity.getSearchRepository().getIndexMapping(entityReference.getType());
Awaitility.await().wait(2000L);
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
List<String> entityIds = new ArrayList<>();
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
entityIds.add(sourceAsMap.get("id").toString());
}
// verify is it present in search
assertTrue(entityIds.contains(entity.getId().toString()));
Assumptions.assumeTrue(supportsSearchIndex);
// create entity
T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS);
EntityReference entityReference = getEntityReference(entity);
IndexMapping indexMapping =
Entity.getSearchRepository().getIndexMapping(entityReference.getType());
waitForEsAsyncOp();
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
List<String> entityIds = new ArrayList<>();
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
entityIds.add(sourceAsMap.get("id").toString());
}
// verify is it present in search
assertTrue(entityIds.contains(entity.getId().toString()));
}
@Test
protected void checkDeletedEntity(TestInfo test)
throws HttpResponseException, InterruptedException {
if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
// create entity
T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS);
EntityReference entityReference = getEntityReference(entity);
IndexMapping indexMapping =
Entity.getSearchRepository().getIndexMapping(entityReference.getType());
Awaitility.await().wait(2000L);
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
List<String> entityIds = new ArrayList<>();
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
entityIds.add(sourceAsMap.get("id").toString());
}
// verify is it present in search
assertTrue(entityIds.contains(entity.getId().toString()));
entityIds.clear();
// delete entity
WebTarget target = getResource(entity.getId());
TestUtils.delete(target, entityClass, ADMIN_AUTH_HEADERS);
// search again in search after deleting
Awaitility.await().wait(2000L);
response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
entityIds.add(sourceAsMap.get("id").toString());
}
// verify if it is deleted from the search as well
assertFalse(entityIds.contains(entity.getId().toString()));
Assumptions.assumeTrue(supportsSearchIndex);
// create entity
T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS);
EntityReference entityReference = getEntityReference(entity);
IndexMapping indexMapping =
Entity.getSearchRepository().getIndexMapping(entityReference.getType());
waitForEsAsyncOp();
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
List<String> entityIds = new ArrayList<>();
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
entityIds.add(sourceAsMap.get("id").toString());
}
// verify is it present in search
assertTrue(entityIds.contains(entity.getId().toString()));
entityIds.clear();
// delete entity
WebTarget target = getResource(entity.getId());
TestUtils.delete(target, entityClass, ADMIN_AUTH_HEADERS);
// search again in search after deleting
waitForEsAsyncOp();
response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
entityIds.add(sourceAsMap.get("id").toString());
}
// verify if it is deleted from the search as well
assertFalse(entityIds.contains(entity.getId().toString()));
}
@Test
protected void updateDescriptionAndCheckInSearch(TestInfo test)
throws IOException, InterruptedException {
if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS);
EntityReference entityReference = getEntityReference(entity);
IndexMapping indexMapping =
Entity.getSearchRepository().getIndexMapping(entityReference.getType());
String desc = "";
String original = JsonUtils.pojoToJson(entity);
entity.setDescription("update description");
entity = patchEntity(entity.getId(), original, entity, ADMIN_AUTH_HEADERS);
Awaitility.await().wait(2000L);
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
if (sourceAsMap.get("id").toString().equals(entity.getId().toString())) {
desc = sourceAsMap.get("description").toString();
break;
}
Assumptions.assumeTrue(supportsSearchIndex);
T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS);
EntityReference entityReference = getEntityReference(entity);
IndexMapping indexMapping =
Entity.getSearchRepository().getIndexMapping(entityReference.getType());
String desc = "";
String original = JsonUtils.pojoToJson(entity);
entity.setDescription("update description");
entity = patchEntity(entity.getId(), original, entity, ADMIN_AUTH_HEADERS);
waitForEsAsyncOp();
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
if (sourceAsMap.get("id").toString().equals(entity.getId().toString())) {
desc = sourceAsMap.get("description").toString();
break;
}
// check if description is updated in search as well
assertEquals(entity.getDescription(), desc);
}
// check if description is updated in search as well
assertEquals(entity.getDescription(), desc);
}
@Test
protected void deleteTagAndCheckRelationshipsInSearch(TestInfo test)
throws HttpResponseException, InterruptedException {
if (supportsTags && supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
// create an entity
T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS);
EntityReference entityReference = getEntityReference(entity);
IndexMapping indexMapping =
Entity.getSearchRepository().getIndexMapping(entityReference.getType());
String origJson = JsonUtils.pojoToJson(entity);
TagResourceTest tagResourceTest = new TagResourceTest();
Tag tag =
tagResourceTest.createEntity(tagResourceTest.createRequest(test), ADMIN_AUTH_HEADERS);
TagLabel tagLabel = EntityUtil.toTagLabel(tag);
entity.setTags(new ArrayList<>());
entity.getTags().add(tagLabel);
List<String> fqnList = new ArrayList<>();
// add tags to entity
entity = patchEntity(entity.getId(), origJson, entity, ADMIN_AUTH_HEADERS);
Awaitility.await().wait(2000L);
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
if (sourceAsMap.get("id").toString().equals(entity.getId().toString())) {
@SuppressWarnings("unchecked")
List<Map<String, String>> listTags = (List<Map<String, String>>) sourceAsMap.get("tags");
listTags.forEach(tempMap -> fqnList.add(tempMap.get("tagFQN")));
break;
}
Assumptions.assumeTrue(supportsSearchIndex && supportsTags);
// create an entity
T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS);
EntityReference entityReference = getEntityReference(entity);
IndexMapping indexMapping =
Entity.getSearchRepository().getIndexMapping(entityReference.getType());
String origJson = JsonUtils.pojoToJson(entity);
TagResourceTest tagResourceTest = new TagResourceTest();
Tag tag = tagResourceTest.createEntity(tagResourceTest.createRequest(test), ADMIN_AUTH_HEADERS);
TagLabel tagLabel = EntityUtil.toTagLabel(tag);
entity.setTags(new ArrayList<>());
entity.getTags().add(tagLabel);
List<String> fqnList = new ArrayList<>();
// add tags to entity
entity = patchEntity(entity.getId(), origJson, entity, ADMIN_AUTH_HEADERS);
waitForEsAsyncOp();
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
if (sourceAsMap.get("id").toString().equals(entity.getId().toString())) {
@SuppressWarnings("unchecked")
List<Map<String, String>> listTags = (List<Map<String, String>>) sourceAsMap.get("tags");
listTags.forEach(tempMap -> fqnList.add(tempMap.get("tagFQN")));
break;
}
// check if the added tag if also added in the entity in search
assertTrue(fqnList.contains(tagLabel.getTagFQN()));
fqnList.clear();
// delete the tag
tagResourceTest.deleteEntity(tag.getId(), false, true, ADMIN_AUTH_HEADERS);
Awaitility.await().wait(2000L);
response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
if (sourceAsMap.get("id").toString().equals(entity.getId().toString())) {
@SuppressWarnings("unchecked")
List<Map<String, String>> listTags = (List<Map<String, String>>) sourceAsMap.get("tags");
listTags.forEach(tempMap -> fqnList.add(tempMap.get("tagFQN")));
break;
}
}
// check if the relationships of tag are also deleted in search
assertFalse(fqnList.contains(tagLabel.getTagFQN()));
}
// check if the added tag if also added in the entity in search
assertTrue(fqnList.contains(tagLabel.getTagFQN()));
fqnList.clear();
// delete the tag
tagResourceTest.deleteEntity(tag.getId(), false, true, ADMIN_AUTH_HEADERS);
waitForEsAsyncOp(500);
response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
if (sourceAsMap.get("id").toString().equals(entity.getId().toString())) {
@SuppressWarnings("unchecked")
List<Map<String, String>> listTags = (List<Map<String, String>>) sourceAsMap.get("tags");
listTags.forEach(tempMap -> fqnList.add(tempMap.get("tagFQN")));
break;
}
}
// check if the relationships of tag are also deleted in search
assertFalse(fqnList.contains(tagLabel.getTagFQN()));
}
@Test
@ -2160,8 +2153,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
throws HttpResponseException {
WebTarget target =
getResource(
String.format(
"elasticsearch/query?q=&index=%s&from=0&deleted=false&size=50", indexName));
String.format("search/query?q=&index=%s&from=0&deleted=false&size=50", indexName));
String result = TestUtils.get(target, String.class, ADMIN_AUTH_HEADERS);
SearchResponse response = null;
try {

50
openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/ReportDataResourceTest.java
View File

@ -10,6 +10,7 @@ import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.INGESTION_BOT_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.TEST_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.TEST_USER_NAME;
import static org.openmetadata.service.util.TestUtils.waitForEsAsyncOp;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -27,8 +28,6 @@ import java.util.UUID;
import javax.ws.rs.client.WebTarget;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.openmetadata.schema.analytics.EntityReportData;
import org.openmetadata.schema.analytics.ReportData;
import org.openmetadata.schema.analytics.WebAnalyticUserActivityReportData;
@ -46,7 +45,7 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest {
private final String collectionName = "analytics/dataInsights/data";
@Test
void report_data_admin_200() throws ParseException, IOException {
void report_data_admin_200() throws ParseException, IOException, InterruptedException {
EntityReportData entityReportData =
new EntityReportData()
.withEntityType("table")
@ -61,24 +60,18 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest {
.withData(entityReportData);
postReportData(reportData, ADMIN_AUTH_HEADERS);
waitForEsAsyncOp(1500);
ResultList<ReportData> reportDataList =
getReportData(
"2022-10-10",
"2022-10-12",
"2022-10-13",
ReportData.ReportDataType.ENTITY_REPORT_DATA,
ADMIN_AUTH_HEADERS);
if (RUN_ELASTIC_SEARCH_TESTCASES) {
String jsonQuery = String.format(JSON_QUERY, "2022-10-10");
assertDocumentCountEquals(jsonQuery, ENTITY_REPORT_DATA_INDEX.value(), 1);
}
assertNotEquals(0, reportDataList.getData().size());
}
@Test
@Execution(ExecutionMode.CONCURRENT)
void report_data_non_auth_user_403() throws ParseException {
EntityReportData entityReportData =
new EntityReportData()
@ -100,8 +93,7 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest {
}
@Test
@Execution(ExecutionMode.CONCURRENT)
void report_data_bot_200() throws HttpResponseException, ParseException {
void report_data_bot_200() throws HttpResponseException, ParseException, InterruptedException {
EntityReportData entityReportData =
new EntityReportData()
.withEntityType("table")
@ -116,11 +108,11 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest {
.withData(entityReportData);
postReportData(reportData, INGESTION_BOT_AUTH_HEADERS);
waitForEsAsyncOp();
ResultList<ReportData> reportDataList =
getReportData(
"2022-10-10",
"2022-10-12",
"2022-10-14",
ReportData.ReportDataType.ENTITY_REPORT_DATA,
INGESTION_BOT_AUTH_HEADERS);
@ -128,7 +120,7 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest {
}
@Test
void delete_endpoint_200() throws ParseException, IOException {
void delete_endpoint_200() throws ParseException, IOException, InterruptedException {
List<ReportData> createReportDataList = new ArrayList<>();
// create some entity report data
@ -178,14 +170,12 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest {
ADMIN_AUTH_HEADERS);
assertNotEquals(0, entityReportDataList.getData().size());
assertNotEquals(0, webAnalyticsReportDataList.getData().size());
if (RUN_ELASTIC_SEARCH_TESTCASES) {
List<String> indices = new ArrayList<>();
indices.add(ENTITY_REPORT_DATA_INDEX.value());
indices.add(WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA_INDEX.value());
for (String index : indices) {
String jsonQuery = String.format(JSON_QUERY, "2022-10-15");
assertDocumentCountEquals(jsonQuery, index, 1);
}
List<String> indices = new ArrayList<>();
indices.add(ENTITY_REPORT_DATA_INDEX.value());
indices.add(WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA_INDEX.value());
for (String index : indices) {
String jsonQuery = String.format(JSON_QUERY, "2022-10-15");
assertDocumentCountEquals(jsonQuery, index, 1);
}
// delete the entity report data and check that it has been deleted
@ -198,11 +188,9 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest {
ReportData.ReportDataType.ENTITY_REPORT_DATA,
ADMIN_AUTH_HEADERS);
assertEquals(0, entityReportDataList.getData().size());
if (RUN_ELASTIC_SEARCH_TESTCASES) {
// Check document has been deleted from elasticsearch
String jsonQuery = String.format(JSON_QUERY, "2022-10-15");
assertDocumentCountEquals(jsonQuery, ENTITY_REPORT_DATA_INDEX.value(), 0);
}
// Check document has been deleted from elasticsearch
String jsonQuery = String.format(JSON_QUERY, "2022-10-15");
assertDocumentCountEquals(jsonQuery, ENTITY_REPORT_DATA_INDEX.value(), 0);
webAnalyticsReportDataList =
getReportData(
"2022-10-15",
@ -252,7 +240,9 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest {
}
private void assertDocumentCountEquals(String query, String index, Integer count)
throws IOException {
throws IOException, InterruptedException {
// async client will return a future which we don't have access to, hence sleep
TestUtils.waitForEsAsyncOp();
JsonNode json = runSearchQuery(query, index);
Integer docCount = json.get("hits").get("total").get("value").asInt();
assertEquals(count, docCount);

2
openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResourceTest.java
View File

@ -42,7 +42,7 @@ public class WebAnalyticEventResourceTest
WebAnalyticEventResource.WebAnalyticEventList.class,
"analytics/web/events",
WebAnalyticEventResource.FIELDS);
supportsSearchIndex = true;
supportsSearchIndex = false;
}
@Test

18
openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java
View File

@ -382,7 +382,7 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
verifyTestCaseResults(testCaseResults, testCase1ResultList, 4);
TestSummary testSummary;
if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
if (supportsSearchIndex) {
testSummary = getTestSummary(null);
assertNotEquals(0, testSummary.getFailed());
assertNotEquals(0, testSummary.getSuccess());
@ -400,7 +400,7 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
testCaseIds.add(testCase1.getId());
testSuiteResourceTest.addTestCasesToLogicalTestSuite(logicalTestSuite, testCaseIds);
if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
if (supportsSearchIndex) {
testSummary = getTestSummary(logicalTestSuite.getId().toString());
assertEquals(1, testSummary.getTotal());
assertEquals(1, testSummary.getFailed());
@ -411,7 +411,7 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
testCaseIds.clear();
testCaseIds.add(testCase.getId());
testSuiteResourceTest.addTestCasesToLogicalTestSuite(logicalTestSuite, testCaseIds);
if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
if (supportsSearchIndex) {
testSummary = getTestSummary(logicalTestSuite.getId().toString());
assertEquals(2, testSummary.getTotal());
}
@ -420,7 +420,7 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
// the summary is updated as expected
deleteLogicalTestCase(logicalTestSuite, testCase.getId());
if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
if (supportsSearchIndex) {
testSummary = getTestSummary(logicalTestSuite.getId().toString());
assertEquals(1, testSummary.getTotal());
}
@ -462,7 +462,7 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
TestSuite testSuite =
testSuiteResourceTest.getEntity(testCase.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS);
if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
if (supportsSearchIndex) {
// test we get the right summary for the executable test suite
TestSummary executableTestSummary =
getTestSummary(testCase.getTestSuite().getId().toString());
@ -471,14 +471,14 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
// test we get the right summary for the logical test suite
if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
if (supportsSearchIndex) {
TestSummary logicalTestSummary = getTestSummary(logicalTestSuite.getId().toString());
assertEquals(1, logicalTestSummary.getTotal());
}
testCaseIds.clear();
testCaseIds.add(testCase.getId());
testSuiteResourceTest.addTestCasesToLogicalTestSuite(logicalTestSuite, testCaseIds);
if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
if (supportsSearchIndex) {
TestSummary logicalTestSummary = getTestSummary(logicalTestSuite.getId().toString());
assertEquals(2, logicalTestSummary.getTotal());
}
@ -486,7 +486,7 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
testSuite =
testSuiteResourceTest.getEntity(testCase.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS);
if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
if (supportsSearchIndex) {
TestSummary executableTestSummary =
getTestSummary(testCase.getTestSuite().getId().toString());
assertEquals(testSuite.getTests().size(), executableTestSummary.getTotal());
@ -497,7 +497,7 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
// cascaded to the logical test suite
deleteLogicalTestCase(logicalTestSuite, testCase.getId());
if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) {
if (supportsSearchIndex) {
TestSummary logicalTestSummary = getTestSummary(logicalTestSuite.getId().toString());
// check the deletion of the test case from the logical test suite is reflected in the summary
assertEquals(1, logicalTestSummary.getTotal());

12
openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java
View File

@ -36,6 +36,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import javax.json.JsonObject;
import javax.json.JsonPatch;
import javax.validation.constraints.Size;
@ -638,4 +639,15 @@ public final class TestUtils {
assertEquals(expected.getIconURL(), actual.getIconURL());
assertEquals(expected.getColor(), actual.getColor());
}
public static void waitForEsAsyncOp() throws InterruptedException {
waitForEsAsyncOp(100);
}
public static void waitForEsAsyncOp(Integer milliseconds) throws InterruptedException {
// Wait for the async operation to complete. We cannot use
// Awaitility here as the test method thread is not
// the owner of the async operation
TimeUnit.MILLISECONDS.sleep(1000);
}
}