diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index b6ffee7c7ad..710b61557cb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -43,7 +43,7 @@ public interface SearchClient { "if((ctx._source.%s == null) || (ctx._source.%s.id == '%s')) { ctx._source.put('%s', params)}"; String SOFT_DELETE_RESTORE_SCRIPT = "ctx._source.put('deleted', '%s')"; String REMOVE_TAGS_CHILDREN_SCRIPT = - "for (int i = 0; i < ctx._source.tags.length; i++) { if (ctx._source.tags[i].tagFQN == '%s') { ctx._source.tags.remove(i) }}"; + "for (int i = 0; i < ctx._source.tags.length; i++) { if (ctx._source.tags[i].tagFQN == params.fqn) { ctx._source.tags.remove(i) }}"; String REMOVE_LINEAGE_SCRIPT = "for (int i = 0; i < ctx._source.lineage.length; i++) { if (ctx._source.lineage[i].doc_id == '%s') { ctx._source.lineage.remove(i) }}"; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 353d93f5133..91771bd2fb4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.io.InputStream; import java.text.ParseException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -532,7 +533,9 @@ public class SearchRepository { case Entity.TAG, Entity.GLOSSARY_TERM -> searchClient.updateChildren( GLOBAL_SEARCH_ALIAS, new ImmutablePair<>("tags.tagFQN", entity.getFullyQualifiedName()), - new ImmutablePair<>(REMOVE_TAGS_CHILDREN_SCRIPT, null)); + new ImmutablePair<>( + REMOVE_TAGS_CHILDREN_SCRIPT, + Collections.singletonMap("fqn", entity.getFullyQualifiedName()))); case Entity.TEST_SUITE -> { TestSuite testSuite = (TestSuite) entity; if (Boolean.TRUE.equals(testSuite.getExecutable())) { diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/query_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/query_index_mapping.json index 06187002bae..5972624dc5d 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/query_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/query_index_mapping.json @@ -302,6 +302,59 @@ "service_suggest": { "type": "completion" }, + "tier": { + "properties": { + "description": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "labelType": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "source": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "state": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + } + } + }, "totalVotes": { "type": "long" }, diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/query_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/query_index_mapping.json index e7e8cebd9fb..fcae659a6f9 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/query_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/query_index_mapping.json @@ -337,6 +337,59 @@ }, "service_suggest": { "type": "completion" + }, + "tier": { + "properties": { + "description": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "labelType": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "source": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "state": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + } + } } } } diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/query_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/query_index_mapping.json index 9a6350e50b1..2197f7c2163 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/query_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/query_index_mapping.json @@ -343,6 +343,59 @@ }, "service_suggest": { "type": "completion" + }, + "tier": { + "properties": { + "description": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "labelType": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "source": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "state": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + } + } } } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java index f402f39a7ce..e3c42d81ec8 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java @@ -17,11 +17,13 @@ import static java.lang.String.format; import static org.openmetadata.service.util.TablesInitializer.validateAndRunSystemDataMigrations; import es.org.elasticsearch.client.RestClient; +import es.org.elasticsearch.client.RestClientBuilder; import io.dropwizard.jersey.jackson.JacksonFeature; import io.dropwizard.testing.ConfigOverride; import io.dropwizard.testing.ResourceHelpers; import io.dropwizard.testing.junit5.DropwizardAppExtension; import java.net.URI; +import java.time.Duration; import java.util.HashSet; import java.util.Set; import javax.ws.rs.client.Client; @@ -29,6 +31,10 @@ import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.WebTarget; import lombok.extern.slf4j.Slf4j; import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; import org.flywaydb.core.Flyway; import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.client.ClientProperties; @@ -41,12 +47,16 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestInstance; import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; +import org.openmetadata.schema.type.IndexMappingLanguage; import org.openmetadata.service.fernet.Fernet; import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator; import org.openmetadata.service.jdbi3.locator.ConnectionType; import org.openmetadata.service.resources.CollectionRegistry; import org.openmetadata.service.resources.events.WebhookCallbackResource; +import org.openmetadata.service.search.SearchRepository; import org.testcontainers.containers.JdbcDatabaseContainer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; import org.testcontainers.elasticsearch.ElasticsearchContainer; @Slf4j @@ -54,6 +64,17 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer; public abstract class OpenMetadataApplicationTest { protected static final String CONFIG_PATH = ResourceHelpers.resourceFilePath("openmetadata-secure-test.yaml"); + public static final String ELASTIC_USER = "elastic"; + public static final String ELASTIC_PASSWORD = "password"; + public static final String ELASTIC_SCHEME = "http"; + public static final Integer ELASTIC_CONNECT_TIMEOUT = 5; + public static final Integer ELASTIC_SOCKET_TIMEOUT = 60; + public static final Integer ELASTIC_KEEP_ALIVE_TIMEOUT = 600; + public static final Integer ELASTIC_BATCH_SIZE = 10; + public static final IndexMappingLanguage ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE = + IndexMappingLanguage.EN; + public static final ElasticSearchConfiguration.SearchType ELASTIC_SEARCH_TYPE = + ElasticSearchConfiguration.SearchType.ELASTICSEARCH; public static DropwizardAppExtension APP; protected static final WebhookCallbackResource webhookCallbackResource = new WebhookCallbackResource(); @@ -61,15 +82,13 @@ public abstract class OpenMetadataApplicationTest { public static Jdbi jdbi; private static ElasticsearchContainer ELASTIC_SEARCH_CONTAINER; - public static final boolean RUN_ELASTIC_SEARCH_TESTCASES = false; - protected static final Set configOverrides = new HashSet<>(); private static final String JDBC_CONTAINER_CLASS_NAME = "org.testcontainers.containers.MySQLContainer"; private static final String JDBC_CONTAINER_IMAGE = "mysql:8"; private static final String ELASTIC_SEARCH_CONTAINER_IMAGE = - "docker.elastic.co/elasticsearch/elasticsearch:7.16.3"; + "docker.elastic.co/elasticsearch/elasticsearch:8.10.2"; private static String HOST; private static String PORT; @@ -140,14 +159,19 @@ public abstract class OpenMetadataApplicationTest { flyway.migrate(); ELASTIC_SEARCH_CONTAINER = new ElasticsearchContainer(elasticSearchContainerImage); - if (RUN_ELASTIC_SEARCH_TESTCASES) { - ELASTIC_SEARCH_CONTAINER.start(); - ELASTIC_SEARCH_CONTAINER.withReuse(true); - String[] parts = ELASTIC_SEARCH_CONTAINER.getHttpHostAddress().split(":"); - HOST = parts[0]; - PORT = parts[1]; - overrideElasticSearchConfig(); - } + ELASTIC_SEARCH_CONTAINER.withPassword("password"); + ELASTIC_SEARCH_CONTAINER.withEnv("discovery.type", "single-node"); + ELASTIC_SEARCH_CONTAINER.withEnv("xpack.security.enabled", "false"); + ELASTIC_SEARCH_CONTAINER.withReuse(false); + ELASTIC_SEARCH_CONTAINER.setWaitStrategy( + new LogMessageWaitStrategy() + .withRegEx(".*(\"message\":\\s?\"started[\\s?|\"].*|] started\n$)") + .withStartupTimeout(Duration.ofMinutes(5))); + ELASTIC_SEARCH_CONTAINER.start(); + String[] parts = ELASTIC_SEARCH_CONTAINER.getHttpHostAddress().split(":"); + HOST = parts[0]; + PORT = parts[1]; + overrideElasticSearchConfig(); overrideDatabaseConfig(sqlContainer); // Migration overrides @@ -172,6 +196,7 @@ public abstract class OpenMetadataApplicationTest { nativeMigrationScriptsLocation, extensionMigrationScripsLocation, false); + createIndices(); APP.before(); createClient(); } @@ -208,9 +233,36 @@ public abstract class OpenMetadataApplicationTest { } } + private void createIndices() { + ElasticSearchConfiguration esConfig = new ElasticSearchConfiguration(); + esConfig + .withHost(HOST) + .withPort(ELASTIC_SEARCH_CONTAINER.getMappedPort(9200)) + .withUsername(ELASTIC_USER) + .withPassword(ELASTIC_PASSWORD) + .withScheme(ELASTIC_SCHEME) + .withConnectionTimeoutSecs(ELASTIC_CONNECT_TIMEOUT) + .withSocketTimeoutSecs(ELASTIC_SOCKET_TIMEOUT) + .withKeepAliveTimeoutSecs(ELASTIC_KEEP_ALIVE_TIMEOUT) + .withBatchSize(ELASTIC_BATCH_SIZE) + .withSearchIndexMappingLanguage(ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE) + .withSearchType(ELASTIC_SEARCH_TYPE); + SearchRepository searchRepository = new SearchRepository(esConfig); + LOG.info("creating indexes."); + searchRepository.createIndexes(); + } + public static RestClient getSearchClient() { - return RestClient.builder(HttpHost.create(ELASTIC_SEARCH_CONTAINER.getHttpHostAddress())) - .build(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials( + AuthScope.ANY, new UsernamePasswordCredentials(ELASTIC_USER, "password")); + + RestClientBuilder builder = + RestClient.builder(HttpHost.create(ELASTIC_SEARCH_CONTAINER.getHttpHostAddress())) + .setHttpClientConfigCallback( + httpClientBuilder -> + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + return builder.build(); } public static WebTarget getResource(String collection) { @@ -231,17 +283,28 @@ public abstract class OpenMetadataApplicationTest { // elastic search overrides configOverrides.add(ConfigOverride.config("elasticsearch.host", HOST)); configOverrides.add(ConfigOverride.config("elasticsearch.port", PORT)); - configOverrides.add(ConfigOverride.config("elasticsearch.scheme", "http")); - configOverrides.add(ConfigOverride.config("elasticsearch.username", "")); - configOverrides.add(ConfigOverride.config("elasticsearch.password", "")); + configOverrides.add(ConfigOverride.config("elasticsearch.scheme", ELASTIC_SCHEME)); + configOverrides.add(ConfigOverride.config("elasticsearch.username", ELASTIC_USER)); + configOverrides.add(ConfigOverride.config("elasticsearch.password", ELASTIC_PASSWORD)); configOverrides.add(ConfigOverride.config("elasticsearch.truststorePath", "")); configOverrides.add(ConfigOverride.config("elasticsearch.truststorePassword", "")); - configOverrides.add(ConfigOverride.config("elasticsearch.connectionTimeoutSecs", "5")); - configOverrides.add(ConfigOverride.config("elasticsearch.socketTimeoutSecs", "60")); - configOverrides.add(ConfigOverride.config("elasticsearch.keepAliveTimeoutSecs", "600")); - configOverrides.add(ConfigOverride.config("elasticsearch.batchSize", "10")); - configOverrides.add(ConfigOverride.config("elasticsearch.searchIndexMappingLanguage", "EN")); - configOverrides.add(ConfigOverride.config("elasticsearch.searchType", "elasticsearch")); + configOverrides.add( + ConfigOverride.config( + "elasticsearch.connectionTimeoutSecs", ELASTIC_CONNECT_TIMEOUT.toString())); + configOverrides.add( + ConfigOverride.config( + "elasticsearch.socketTimeoutSecs", ELASTIC_SOCKET_TIMEOUT.toString())); + configOverrides.add( + ConfigOverride.config( + "elasticsearch.keepAliveTimeoutSecs", ELASTIC_KEEP_ALIVE_TIMEOUT.toString())); + configOverrides.add( + ConfigOverride.config("elasticsearch.batchSize", ELASTIC_BATCH_SIZE.toString())); + configOverrides.add( + ConfigOverride.config( + "elasticsearch.searchIndexMappingLanguage", + ELASTIC_SEARCH_INDEX_MAPPING_LANGUAGE.value())); + configOverrides.add( + ConfigOverride.config("elasticsearch.searchType", ELASTIC_SEARCH_TYPE.value())); } private static void overrideDatabaseConfig(JdbcDatabaseContainer sqlContainer) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index bb88983337e..ffa81ddce2d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -1927,170 +1927,163 @@ public abstract class EntityResourceTest indexNamesFromResponse = new ArrayList<>(); - for (int i = 0; i < jsonArray.length(); i++) { - JSONObject jsonObject = jsonArray.getJSONObject(i); - String indexName = jsonObject.getString("index"); - indexNamesFromResponse.add(indexName); - } - client.close(); + RestClient client = getSearchClient(); + Request request = new Request("GET", "/_cat/indices"); + request.addParameter("format", "json"); + Response response = client.performRequest(request); + JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity())); + List indexNamesFromResponse = new ArrayList<>(); + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + String indexName = jsonObject.getString("index"); + indexNamesFromResponse.add(indexName); } + client.close(); } @Test protected void checkCreatedEntity(TestInfo test) throws IOException, InterruptedException { - if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { - // create entity - T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS); - EntityReference entityReference = getEntityReference(entity); - IndexMapping indexMapping = - Entity.getSearchRepository().getIndexMapping(entityReference.getType()); - Awaitility.await().wait(2000L); - SearchResponse response = - getResponseFormSearch( - indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias())); - List entityIds = new ArrayList<>(); - SearchHit[] hits = response.getHits().getHits(); - for (SearchHit hit : hits) { - Map sourceAsMap = hit.getSourceAsMap(); - entityIds.add(sourceAsMap.get("id").toString()); - } - // verify is it present in search - assertTrue(entityIds.contains(entity.getId().toString())); + Assumptions.assumeTrue(supportsSearchIndex); + // create entity + T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS); + EntityReference entityReference = getEntityReference(entity); + IndexMapping indexMapping = + Entity.getSearchRepository().getIndexMapping(entityReference.getType()); + waitForEsAsyncOp(); + SearchResponse response = + getResponseFormSearch( + indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias())); + List entityIds = new ArrayList<>(); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + Map sourceAsMap = hit.getSourceAsMap(); + entityIds.add(sourceAsMap.get("id").toString()); } + // verify is it present in search + assertTrue(entityIds.contains(entity.getId().toString())); } @Test protected void checkDeletedEntity(TestInfo test) throws HttpResponseException, InterruptedException { - if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { - // create entity - T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS); - EntityReference entityReference = getEntityReference(entity); - IndexMapping indexMapping = - Entity.getSearchRepository().getIndexMapping(entityReference.getType()); - Awaitility.await().wait(2000L); - SearchResponse response = - getResponseFormSearch( - indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias())); - List entityIds = new ArrayList<>(); - SearchHit[] hits = response.getHits().getHits(); - for (SearchHit hit : hits) { - Map sourceAsMap = hit.getSourceAsMap(); - entityIds.add(sourceAsMap.get("id").toString()); - } - // verify is it present in search - assertTrue(entityIds.contains(entity.getId().toString())); - entityIds.clear(); - // delete entity - WebTarget target = getResource(entity.getId()); - TestUtils.delete(target, entityClass, ADMIN_AUTH_HEADERS); - // search again in search after deleting - - Awaitility.await().wait(2000L); - response = - getResponseFormSearch( - indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias())); - hits = response.getHits().getHits(); - for (SearchHit hit : hits) { - Map sourceAsMap = hit.getSourceAsMap(); - entityIds.add(sourceAsMap.get("id").toString()); - } - // verify if it is deleted from the search as well - assertFalse(entityIds.contains(entity.getId().toString())); + Assumptions.assumeTrue(supportsSearchIndex); + // create entity + T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS); + EntityReference entityReference = getEntityReference(entity); + IndexMapping indexMapping = + Entity.getSearchRepository().getIndexMapping(entityReference.getType()); + waitForEsAsyncOp(); + SearchResponse response = + getResponseFormSearch( + indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias())); + List entityIds = new ArrayList<>(); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + Map sourceAsMap = hit.getSourceAsMap(); + entityIds.add(sourceAsMap.get("id").toString()); } + // verify is it present in search + assertTrue(entityIds.contains(entity.getId().toString())); + entityIds.clear(); + // delete entity + WebTarget target = getResource(entity.getId()); + TestUtils.delete(target, entityClass, ADMIN_AUTH_HEADERS); + // search again in search after deleting + + waitForEsAsyncOp(); + response = + getResponseFormSearch( + indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias())); + hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + Map sourceAsMap = hit.getSourceAsMap(); + entityIds.add(sourceAsMap.get("id").toString()); + } + // verify if it is deleted from the search as well + assertFalse(entityIds.contains(entity.getId().toString())); } @Test protected void updateDescriptionAndCheckInSearch(TestInfo test) throws IOException, InterruptedException { - if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { - T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS); - EntityReference entityReference = getEntityReference(entity); - IndexMapping indexMapping = - Entity.getSearchRepository().getIndexMapping(entityReference.getType()); - String desc = ""; - String original = JsonUtils.pojoToJson(entity); - entity.setDescription("update description"); - entity = patchEntity(entity.getId(), original, entity, ADMIN_AUTH_HEADERS); - Awaitility.await().wait(2000L); - SearchResponse response = - getResponseFormSearch( - indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias())); - SearchHit[] hits = response.getHits().getHits(); - for (SearchHit hit : hits) { - Map sourceAsMap = hit.getSourceAsMap(); - if (sourceAsMap.get("id").toString().equals(entity.getId().toString())) { - desc = sourceAsMap.get("description").toString(); - break; - } + Assumptions.assumeTrue(supportsSearchIndex); + T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS); + EntityReference entityReference = getEntityReference(entity); + IndexMapping indexMapping = + Entity.getSearchRepository().getIndexMapping(entityReference.getType()); + String desc = ""; + String original = JsonUtils.pojoToJson(entity); + entity.setDescription("update description"); + entity = patchEntity(entity.getId(), original, entity, ADMIN_AUTH_HEADERS); + waitForEsAsyncOp(); + SearchResponse response = + getResponseFormSearch( + indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias())); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + Map sourceAsMap = hit.getSourceAsMap(); + if (sourceAsMap.get("id").toString().equals(entity.getId().toString())) { + desc = sourceAsMap.get("description").toString(); + break; } - // check if description is updated in search as well - assertEquals(entity.getDescription(), desc); } + // check if description is updated in search as well + assertEquals(entity.getDescription(), desc); } @Test protected void deleteTagAndCheckRelationshipsInSearch(TestInfo test) throws HttpResponseException, InterruptedException { - if (supportsTags && supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { - // create an entity - T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS); - EntityReference entityReference = getEntityReference(entity); - IndexMapping indexMapping = - Entity.getSearchRepository().getIndexMapping(entityReference.getType()); - String origJson = JsonUtils.pojoToJson(entity); - TagResourceTest tagResourceTest = new TagResourceTest(); - Tag tag = - tagResourceTest.createEntity(tagResourceTest.createRequest(test), ADMIN_AUTH_HEADERS); - TagLabel tagLabel = EntityUtil.toTagLabel(tag); - entity.setTags(new ArrayList<>()); - entity.getTags().add(tagLabel); - List fqnList = new ArrayList<>(); - // add tags to entity - entity = patchEntity(entity.getId(), origJson, entity, ADMIN_AUTH_HEADERS); - Awaitility.await().wait(2000L); - SearchResponse response = - getResponseFormSearch( - indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias())); - SearchHit[] hits = response.getHits().getHits(); - for (SearchHit hit : hits) { - Map sourceAsMap = hit.getSourceAsMap(); - if (sourceAsMap.get("id").toString().equals(entity.getId().toString())) { - @SuppressWarnings("unchecked") - List> listTags = (List>) sourceAsMap.get("tags"); - listTags.forEach(tempMap -> fqnList.add(tempMap.get("tagFQN"))); - break; - } + Assumptions.assumeTrue(supportsSearchIndex && supportsTags); + // create an entity + T entity = createEntity(createRequest(test), ADMIN_AUTH_HEADERS); + EntityReference entityReference = getEntityReference(entity); + IndexMapping indexMapping = + Entity.getSearchRepository().getIndexMapping(entityReference.getType()); + String origJson = JsonUtils.pojoToJson(entity); + TagResourceTest tagResourceTest = new TagResourceTest(); + Tag tag = tagResourceTest.createEntity(tagResourceTest.createRequest(test), ADMIN_AUTH_HEADERS); + TagLabel tagLabel = EntityUtil.toTagLabel(tag); + entity.setTags(new ArrayList<>()); + entity.getTags().add(tagLabel); + List fqnList = new ArrayList<>(); + // add tags to entity + entity = patchEntity(entity.getId(), origJson, entity, ADMIN_AUTH_HEADERS); + waitForEsAsyncOp(); + SearchResponse response = + getResponseFormSearch( + indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias())); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + Map sourceAsMap = hit.getSourceAsMap(); + if (sourceAsMap.get("id").toString().equals(entity.getId().toString())) { + @SuppressWarnings("unchecked") + List> listTags = (List>) sourceAsMap.get("tags"); + listTags.forEach(tempMap -> fqnList.add(tempMap.get("tagFQN"))); + break; } - // check if the added tag if also added in the entity in search - assertTrue(fqnList.contains(tagLabel.getTagFQN())); - fqnList.clear(); - // delete the tag - tagResourceTest.deleteEntity(tag.getId(), false, true, ADMIN_AUTH_HEADERS); - Awaitility.await().wait(2000L); - response = - getResponseFormSearch( - indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias())); - hits = response.getHits().getHits(); - for (SearchHit hit : hits) { - Map sourceAsMap = hit.getSourceAsMap(); - if (sourceAsMap.get("id").toString().equals(entity.getId().toString())) { - @SuppressWarnings("unchecked") - List> listTags = (List>) sourceAsMap.get("tags"); - listTags.forEach(tempMap -> fqnList.add(tempMap.get("tagFQN"))); - break; - } - } - // check if the relationships of tag are also deleted in search - assertFalse(fqnList.contains(tagLabel.getTagFQN())); } + // check if the added tag if also added in the entity in search + assertTrue(fqnList.contains(tagLabel.getTagFQN())); + fqnList.clear(); + // delete the tag + tagResourceTest.deleteEntity(tag.getId(), false, true, ADMIN_AUTH_HEADERS); + waitForEsAsyncOp(500); + response = + getResponseFormSearch( + indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias())); + hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + Map sourceAsMap = hit.getSourceAsMap(); + if (sourceAsMap.get("id").toString().equals(entity.getId().toString())) { + @SuppressWarnings("unchecked") + List> listTags = (List>) sourceAsMap.get("tags"); + listTags.forEach(tempMap -> fqnList.add(tempMap.get("tagFQN"))); + break; + } + } + // check if the relationships of tag are also deleted in search + assertFalse(fqnList.contains(tagLabel.getTagFQN())); } @Test @@ -2160,8 +2153,7 @@ public abstract class EntityResourceTest reportDataList = getReportData( "2022-10-10", - "2022-10-12", + "2022-10-13", ReportData.ReportDataType.ENTITY_REPORT_DATA, ADMIN_AUTH_HEADERS); - if (RUN_ELASTIC_SEARCH_TESTCASES) { - String jsonQuery = String.format(JSON_QUERY, "2022-10-10"); - assertDocumentCountEquals(jsonQuery, ENTITY_REPORT_DATA_INDEX.value(), 1); - } - assertNotEquals(0, reportDataList.getData().size()); } @Test - @Execution(ExecutionMode.CONCURRENT) void report_data_non_auth_user_403() throws ParseException { EntityReportData entityReportData = new EntityReportData() @@ -100,8 +93,7 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest { } @Test - @Execution(ExecutionMode.CONCURRENT) - void report_data_bot_200() throws HttpResponseException, ParseException { + void report_data_bot_200() throws HttpResponseException, ParseException, InterruptedException { EntityReportData entityReportData = new EntityReportData() .withEntityType("table") @@ -116,11 +108,11 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest { .withData(entityReportData); postReportData(reportData, INGESTION_BOT_AUTH_HEADERS); - + waitForEsAsyncOp(); ResultList reportDataList = getReportData( - "2022-10-10", "2022-10-12", + "2022-10-14", ReportData.ReportDataType.ENTITY_REPORT_DATA, INGESTION_BOT_AUTH_HEADERS); @@ -128,7 +120,7 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest { } @Test - void delete_endpoint_200() throws ParseException, IOException { + void delete_endpoint_200() throws ParseException, IOException, InterruptedException { List createReportDataList = new ArrayList<>(); // create some entity report data @@ -178,14 +170,12 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest { ADMIN_AUTH_HEADERS); assertNotEquals(0, entityReportDataList.getData().size()); assertNotEquals(0, webAnalyticsReportDataList.getData().size()); - if (RUN_ELASTIC_SEARCH_TESTCASES) { - List indices = new ArrayList<>(); - indices.add(ENTITY_REPORT_DATA_INDEX.value()); - indices.add(WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA_INDEX.value()); - for (String index : indices) { - String jsonQuery = String.format(JSON_QUERY, "2022-10-15"); - assertDocumentCountEquals(jsonQuery, index, 1); - } + List indices = new ArrayList<>(); + indices.add(ENTITY_REPORT_DATA_INDEX.value()); + indices.add(WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA_INDEX.value()); + for (String index : indices) { + String jsonQuery = String.format(JSON_QUERY, "2022-10-15"); + assertDocumentCountEquals(jsonQuery, index, 1); } // delete the entity report data and check that it has been deleted @@ -198,11 +188,9 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest { ReportData.ReportDataType.ENTITY_REPORT_DATA, ADMIN_AUTH_HEADERS); assertEquals(0, entityReportDataList.getData().size()); - if (RUN_ELASTIC_SEARCH_TESTCASES) { - // Check document has been deleted from elasticsearch - String jsonQuery = String.format(JSON_QUERY, "2022-10-15"); - assertDocumentCountEquals(jsonQuery, ENTITY_REPORT_DATA_INDEX.value(), 0); - } + // Check document has been deleted from elasticsearch + String jsonQuery = String.format(JSON_QUERY, "2022-10-15"); + assertDocumentCountEquals(jsonQuery, ENTITY_REPORT_DATA_INDEX.value(), 0); webAnalyticsReportDataList = getReportData( "2022-10-15", @@ -252,7 +240,9 @@ class ReportDataResourceTest extends OpenMetadataApplicationTest { } private void assertDocumentCountEquals(String query, String index, Integer count) - throws IOException { + throws IOException, InterruptedException { + // async client will return a future which we don't have access to, hence sleep + TestUtils.waitForEsAsyncOp(); JsonNode json = runSearchQuery(query, index); Integer docCount = json.get("hits").get("total").get("value").asInt(); assertEquals(count, docCount); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResourceTest.java index a7e5788c1bf..ede53840bba 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResourceTest.java @@ -42,7 +42,7 @@ public class WebAnalyticEventResourceTest WebAnalyticEventResource.WebAnalyticEventList.class, "analytics/web/events", WebAnalyticEventResource.FIELDS); - supportsSearchIndex = true; + supportsSearchIndex = false; } @Test diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index cf9e80d5ff3..299b783a29a 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -382,7 +382,7 @@ public class TestCaseResourceTest extends EntityResourceTest