From a7ef0a82b1df4ec103fcbf8f2573ed907483060f Mon Sep 17 00:00:00 2001 From: 07Himank <112613760+07Himank@users.noreply.github.com> Date: Thu, 29 Jun 2023 11:38:10 +0530 Subject: [PATCH] Test suite index task(addded test_suite_search_index) (#12184) * added search test case index * checkstyle * working on adding test suite index * addes test_suite_search_index * removed comments * improvemtn * fixes * added test_suite_index_mapping for jp and zh languages * check style --- .../ElasticSearchIndexDefinition.java | 7 +- .../ElasticSearchIndexFactory.java | 5 +- .../elasticsearch/indexes/TestSuiteIndex.java | 18 ++++ .../ElasticSearchClientImpl.java | 43 +++++--- .../openSearch/OpenSearchClientImpl.java | 43 +++++--- .../en/test_suite_index_mapping.json | 92 ++++++++++++++++ .../jp/test_suite_index_mapping.json | 102 ++++++++++++++++++ .../zh/test_suite_index_mapping.json | 92 ++++++++++++++++ 8 files changed, 366 insertions(+), 36 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/indexes/TestSuiteIndex.java create mode 100644 openmetadata-service/src/main/resources/elasticsearch/en/test_suite_index_mapping.json create mode 100644 openmetadata-service/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json create mode 100644 openmetadata-service/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexDefinition.java b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexDefinition.java index 41cbb5567f4..62d880c6be3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexDefinition.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexDefinition.java @@ -76,6 +76,9 @@ public class ElasticSearchIndexDefinition { ENTITY_REPORT_DATA, "entity_report_data_index", "/elasticsearch/entity_report_data_index.json"), TEST_CASE_SEARCH_INDEX( Entity.TEST_CASE, "test_case_search_index", "/elasticsearch/%s/test_case_index_mapping.json"), + + TEST_SUITE_SEARCH_INDEX( + Entity.TEST_SUITE, "test_suite_search_index", "/elasticsearch/%s/test_suite_index_mapping.json"), WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA_INDEX( Entity.WEB_ANALYTIC_EVENT, "web_analytic_entity_view_report_data_index", @@ -171,8 +174,10 @@ public class ElasticSearchIndexDefinition { return ElasticSearchIndexType.CONTAINER_SEARCH_INDEX; } else if (type.equalsIgnoreCase(Entity.QUERY)) { return ElasticSearchIndexType.QUERY_SEARCH_INDEX; - } else if (type.equalsIgnoreCase(Entity.TEST_SUITE) || type.equalsIgnoreCase(Entity.TEST_CASE)) { + } else if (type.equalsIgnoreCase(Entity.TEST_CASE)) { return ElasticSearchIndexType.TEST_CASE_SEARCH_INDEX; + } else if (type.equalsIgnoreCase(Entity.TEST_SUITE)) { + return ElasticSearchIndexType.TEST_SUITE_SEARCH_INDEX; } throw new EventPublisherException("Failed to find index doc for type " + type); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexFactory.java index 305ee6e21ea..dba25d5c1ef 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexFactory.java @@ -13,6 +13,7 @@ import org.openmetadata.schema.entity.data.Topic; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.service.Entity; import org.openmetadata.service.elasticsearch.indexes.ContainerIndex; import org.openmetadata.service.elasticsearch.indexes.DashboardIndex; @@ -24,6 +25,7 @@ import org.openmetadata.service.elasticsearch.indexes.QueryIndex; import org.openmetadata.service.elasticsearch.indexes.TableIndex; import org.openmetadata.service.elasticsearch.indexes.TagIndex; import org.openmetadata.service.elasticsearch.indexes.TeamIndex; +import org.openmetadata.service.elasticsearch.indexes.TestSuiteIndex; import org.openmetadata.service.elasticsearch.indexes.TopicIndex; import org.openmetadata.service.elasticsearch.indexes.UserIndex; @@ -56,8 +58,9 @@ public class ElasticSearchIndexFactory { case Entity.CONTAINER: return new ContainerIndex((Container) entity); case Entity.TEST_CASE: - case Entity.TEST_SUITE: return new TestCaseIndex((TestCase) entity); + case Entity.TEST_SUITE: + return new TestSuiteIndex((TestSuite) entity); default: LOG.warn("Ignoring Entity Type {}", entityType); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/indexes/TestSuiteIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/indexes/TestSuiteIndex.java new file mode 100644 index 00000000000..d3f95670f9c --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/indexes/TestSuiteIndex.java @@ -0,0 +1,18 @@ +package org.openmetadata.service.elasticsearch.indexes; + +import java.util.Map; +import org.openmetadata.schema.tests.TestSuite; +import org.openmetadata.service.util.JsonUtils; + +public class TestSuiteIndex implements ElasticSearchIndex { + TestSuite testSuite; + + public TestSuiteIndex(TestSuite testSuite) { + this.testSuite = testSuite; + } + + public Map buildESDoc() { + Map doc = JsonUtils.getMap(testSuite); + return doc; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/ElasticSearchClientImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/ElasticSearchClientImpl.java index f990264f693..9b77d683b4f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/ElasticSearchClientImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/ElasticSearchClientImpl.java @@ -133,6 +133,7 @@ import org.openmetadata.service.elasticsearch.indexes.ElasticSearchIndex; import org.openmetadata.service.elasticsearch.indexes.GlossaryTermIndex; import org.openmetadata.service.elasticsearch.indexes.TagIndex; import org.openmetadata.service.elasticsearch.indexes.TeamIndex; +import org.openmetadata.service.elasticsearch.indexes.TestSuiteIndex; import org.openmetadata.service.elasticsearch.indexes.UserIndex; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.DataInsightChartRepository; @@ -279,6 +280,7 @@ public class ElasticSearchClientImpl implements SearchClient { searchSourceBuilder = buildQuerySearchBuilder(request.getQuery(), request.getFrom(), request.getSize()); break; case "test_case_search_index": + case "test_suite_search_index": searchSourceBuilder = buildTestCaseSearch(request.getQuery(), request.getFrom(), request.getSize()); break; default: @@ -1178,26 +1180,33 @@ public class ElasticSearchClientImpl implements SearchClient { @Override public void updateTestSuite(ChangeEvent event) throws IOException { ElasticSearchIndexDefinition.ElasticSearchIndexType indexType = - ElasticSearchIndexDefinition.getIndexMappingByEntityType(Entity.TEST_CASE); + ElasticSearchIndexDefinition.getIndexMappingByEntityType(Entity.TEST_SUITE); TestSuite testSuite = (TestSuite) event.getEntity(); UUID testSuiteId = testSuite.getId(); - if (event.getEventType() == ENTITY_DELETED) { - if (testSuite.getExecutable()) { - DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(indexType.indexName); - deleteByQueryRequest.setQuery(new MatchQueryBuilder("testSuites.id", testSuiteId.toString())); - deleteEntityFromElasticSearchByQuery(deleteByQueryRequest); - } else { - UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexType.indexName); - updateByQueryRequest.setQuery(new MatchQueryBuilder("testSuites.id", testSuiteId.toString())); - String scriptTxt = - "for (int i = 0; i < ctx._source.testSuites.length; i++) { if (ctx._source.testSuites[i].id == '%s') { ctx._source.testSuites.remove(i) }}"; - Script script = - new Script( - ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, String.format(scriptTxt, testSuiteId), new HashMap<>()); - updateByQueryRequest.setScript(script); - updateElasticSearchByQuery(updateByQueryRequest); - } + UpdateRequest updateRequest = new UpdateRequest(indexType.indexName, testSuiteId.toString()); + TestSuiteIndex testSuiteIndex; + + switch (event.getEventType()) { + case ENTITY_CREATED: + testSuiteIndex = new TestSuiteIndex((TestSuite) event.getEntity()); + updateRequest.doc(JsonUtils.pojoToJson(testSuiteIndex.buildESDoc()), XContentType.JSON); + updateRequest.docAsUpsert(true); + updateElasticSearch(updateRequest); + break; + case ENTITY_UPDATED: + testSuiteIndex = new TestSuiteIndex((TestSuite) event.getEntity()); + scriptedUpsert(testSuiteIndex.buildESDoc(), updateRequest); + updateElasticSearch(updateRequest); + break; + case ENTITY_SOFT_DELETED: + softDeleteEntity(updateRequest); + updateElasticSearch(updateRequest); + break; + case ENTITY_DELETED: + DeleteRequest deleteRequest = new DeleteRequest(indexType.indexName, event.getEntityId().toString()); + deleteEntityFromElasticSearch(deleteRequest); + break; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OpenSearchClientImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OpenSearchClientImpl.java index 7e039099c49..aab2c229060 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OpenSearchClientImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OpenSearchClientImpl.java @@ -68,6 +68,7 @@ import org.openmetadata.service.elasticsearch.indexes.ElasticSearchIndex; import org.openmetadata.service.elasticsearch.indexes.GlossaryTermIndex; import org.openmetadata.service.elasticsearch.indexes.TagIndex; import org.openmetadata.service.elasticsearch.indexes.TeamIndex; +import org.openmetadata.service.elasticsearch.indexes.TestSuiteIndex; import org.openmetadata.service.elasticsearch.indexes.UserIndex; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.DataInsightChartRepository; @@ -279,6 +280,7 @@ public class OpenSearchClientImpl implements SearchClient { searchSourceBuilder = buildQuerySearchBuilder(request.getQuery(), request.getFrom(), request.getSize()); break; case "test_case_search_index": + case "test_suite_search_index": searchSourceBuilder = buildTestCaseSearch(request.getQuery(), request.getFrom(), request.getSize()); break; default: @@ -1172,26 +1174,33 @@ public class OpenSearchClientImpl implements SearchClient { @Override public void updateTestSuite(ChangeEvent event) throws IOException { ElasticSearchIndexDefinition.ElasticSearchIndexType indexType = - ElasticSearchIndexDefinition.getIndexMappingByEntityType(Entity.TEST_CASE); + ElasticSearchIndexDefinition.getIndexMappingByEntityType(Entity.TEST_SUITE); TestSuite testSuite = (TestSuite) event.getEntity(); UUID testSuiteId = testSuite.getId(); - if (event.getEventType() == ENTITY_DELETED) { - if (Boolean.TRUE.equals(testSuite.getExecutable())) { - DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(indexType.indexName); - deleteByQueryRequest.setQuery(new MatchQueryBuilder("testSuites.id", testSuiteId.toString())); - deleteEntityFromElasticSearchByQuery(deleteByQueryRequest); - } else { - UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexType.indexName); - updateByQueryRequest.setQuery(new MatchQueryBuilder("testSuites.id", testSuiteId.toString())); - String scriptTxt = - "for (int i = 0; i < ctx._source.testSuites.length; i++) { if (ctx._source.testSuites[i].id == '%s') { ctx._source.testSuites.remove(i) }}"; - Script script = - new Script( - ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, String.format(scriptTxt, testSuiteId), new HashMap<>()); - updateByQueryRequest.setScript(script); - updateElasticSearchByQuery(updateByQueryRequest); - } + UpdateRequest updateRequest = new UpdateRequest(indexType.indexName, testSuiteId.toString()); + TestSuiteIndex testSuiteIndex; + + switch (event.getEventType()) { + case ENTITY_CREATED: + testSuiteIndex = new TestSuiteIndex((TestSuite) event.getEntity()); + updateRequest.doc(JsonUtils.pojoToJson(testSuiteIndex.buildESDoc()), XContentType.JSON); + updateRequest.docAsUpsert(true); + updateElasticSearch(updateRequest); + break; + case ENTITY_UPDATED: + testSuiteIndex = new TestSuiteIndex((TestSuite) event.getEntity()); + scriptedUpsert(testSuiteIndex.buildESDoc(), updateRequest); + updateElasticSearch(updateRequest); + break; + case ENTITY_SOFT_DELETED: + softDeleteEntity(updateRequest); + updateElasticSearch(updateRequest); + break; + case ENTITY_DELETED: + DeleteRequest deleteRequest = new DeleteRequest(indexType.indexName, event.getEntityId().toString()); + deleteEntityFromElasticSearch(deleteRequest); + break; } } diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/test_suite_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/test_suite_index_mapping.json new file mode 100644 index 00000000000..acf63fef3b3 --- /dev/null +++ b/openmetadata-service/src/main/resources/elasticsearch/en/test_suite_index_mapping.json @@ -0,0 +1,92 @@ +{ + "settings": { + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, + "om_ngram": { + "tokenizer": "ngram", + "min_gram": 2, + "max_gram": 3, + "filter": [ + "lowercase" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + } + } + } + }, + "mappings": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "description": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + }, + "executable": { + "type": "text" + } + } + } +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json new file mode 100644 index 00000000000..2d76b02ce97 --- /dev/null +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json @@ -0,0 +1,102 @@ +{ + "settings": { + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, + "om_analyzer_jp" : { + "tokenizer" : "kuromoji_tokenizer", + "type" : "custom", + "filter" : [ + "kuromoji_baseform", + "kuromoji_part_of_speech", + "kuromoji_number", + "kuromoji_stemmer" + ] + }, + "om_ngram": { + "tokenizer": "ngram", + "min_gram": 1, + "max_gram": 2, + "filter": [ + "lowercase" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + } + } + } + }, + "mappings": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "text", + "analyzer": "om_analyzer_jp", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "description": { + "type": "text", + "analyzer": "om_analyzer_jp", + "fields": { + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer_jp", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + }, + "executable": { + "type": "text" + } + } + } +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json new file mode 100644 index 00000000000..cb5466c171a --- /dev/null +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json @@ -0,0 +1,92 @@ +{ + "settings": { + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, + "om_ngram": { + "tokenizer": "ngram", + "min_gram": 2, + "max_gram": 3, + "filter": [ + "lowercase" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + } + } + } + }, + "mappings": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "text", + "analyzer": "ik_max_word", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "description": { + "type": "text", + "analyzer": "ik_max_word", + "fields": { + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "displayName": { + "type": "text", + "analyzer": "ik_max_word", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + }, + "executable": { + "type": "text" + } + } + } +} \ No newline at end of file