From db921645f3011144baafcb58e4b94dae03ff96e4 Mon Sep 17 00:00:00 2001 From: 07Himank <112613760+07Himank@users.noreply.github.com> Date: Mon, 11 Dec 2023 16:10:52 +0530 Subject: [PATCH] changes needed for adding knowledgeCenterIndex (#14223) * chnages needed for adding knowledgeCenterIndex * fix tests * CI fix --- .../java/org/openmetadata/service/Entity.java | 9 +++++++ .../service/OpenMetadataApplication.java | 3 ++- .../service/resources/apps/AppResource.java | 3 ++- .../service/search/SearchIndexFactory.java | 8 +++--- .../service/search/SearchRepository.java | 26 +++++++++++++++---- .../elasticsearch/ElasticSearchClient.java | 1 + .../ElasticSearchEntitiesProcessor.java | 3 +-- .../OpenSearchEntitiesProcessor.java | 3 +-- .../service/util/TablesInitializer.java | 4 ++- 9 files changed, 45 insertions(+), 15 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index d3118ee574f..0b68876427b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -35,6 +35,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.UUID; +import javax.ws.rs.BadRequestException; import javax.ws.rs.core.UriInfo; import lombok.Getter; import lombok.NonNull; @@ -63,6 +64,7 @@ import org.openmetadata.service.jdbi3.TokenRepository; import org.openmetadata.service.jdbi3.UsageRepository; import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; import org.openmetadata.service.search.SearchRepository; +import org.openmetadata.service.search.indexes.SearchIndex; import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.FullyQualifiedName; @@ -555,4 +557,11 @@ public final class Entity { } } } + + public static SearchIndex buildSearchIndex(String entityType, Object entity) { + if (searchRepository != null) { + return searchRepository.getSearchIndexFactory().buildIndex(entityType, entity); + } + throw new BadRequestException("searchrepository not initialized"); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index e4bd4482070..02dc3b6c130 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -90,6 +90,7 @@ import org.openmetadata.service.monitoring.EventMonitorPublisher; import org.openmetadata.service.resources.CollectionRegistry; import org.openmetadata.service.resources.databases.DatasourceConfig; import org.openmetadata.service.resources.settings.SettingsCache; +import org.openmetadata.service.search.SearchIndexFactory; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.secrets.SecretsManager; import org.openmetadata.service.secrets.SecretsManagerFactory; @@ -138,7 +139,7 @@ public class OpenMetadataApplication extends Application { // Create an On Demand DAO CollectionDAO dao = Entity.getCollectionDAO(); - searchRepository = new SearchRepository(config.getElasticSearchConfiguration()); + searchRepository = new SearchRepository(config.getElasticSearchConfiguration(), new SearchIndexFactory()); try { AppScheduler.initialize(dao, searchRepository); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java index 9fed9cd8a0c..e7c4191d7d4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java @@ -73,9 +73,8 @@ import org.openmetadata.service.search.indexes.WebAnalyticUserActivityReportData @Slf4j public class SearchIndexFactory { - private SearchIndexFactory() {} - public static SearchIndex buildIndex(String entityType, Object entity) { + public SearchIndex buildIndex(String entityType, Object entity) { switch (entityType) { case Entity.TABLE: return new TableIndex((Table) entity); @@ -150,8 +149,11 @@ public class SearchIndexFactory { case Entity.TEST_CASE_RESOLUTION_STATUS: return new TestCaseResolutionStatusIndex((TestCaseResolutionStatus) entity); default: - LOG.warn("Ignoring Entity Type {}", entityType); + return buildExternalIndexes(entityType, entity); } + } + + protected SearchIndex buildExternalIndexes(String entityType, Object entity) { throw new IllegalArgumentException(String.format("Entity Type [%s] is not valid for Index Factory", entityType)); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index d9986b3b8e2..1d6ae0db8b9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -36,6 +36,7 @@ import java.util.SortedMap; import javax.json.JsonObject; import javax.ws.rs.core.Response; import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -69,6 +70,8 @@ public class SearchRepository { private final String language; + @Getter @Setter private SearchIndexFactory searchIndexFactory; + private final List inheritableFields = List.of(Entity.FIELD_OWNER, Entity.FIELD_DOMAIN, Entity.FIELD_DISABLED); private final List propagateFields = List.of(Entity.FIELD_TAGS); @@ -85,13 +88,14 @@ public class SearchRepository { public static final String ELASTIC_SEARCH_EXTENSION = "service.eventPublisher"; - public SearchRepository(ElasticSearchConfiguration config) { + public SearchRepository(ElasticSearchConfiguration config, SearchIndexFactory searchIndexFactory) { elasticSearchConfiguration = config; if (config != null && config.getSearchType() == ElasticSearchConfiguration.SearchType.OPENSEARCH) { searchClient = new OpenSearchClient(config); } else { searchClient = new ElasticSearchClient(config); } + this.searchIndexFactory = searchIndexFactory; this.language = config != null ? config.getSearchIndexMappingLanguage().value() : "en"; loadIndexMappings(); Entity.setSearchRepository(this); @@ -106,17 +110,29 @@ public class SearchRepository { } private void loadIndexMappings() { + Set entities; entityIndexMap = new HashMap<>(); try (InputStream in = getClass().getResourceAsStream("/elasticsearch/indexMapping.json")) { assert in != null; JsonObject jsonPayload = JsonUtils.readJson(new String(in.readAllBytes())).asJsonObject(); - Set entities = jsonPayload.keySet(); + entities = jsonPayload.keySet(); for (String s : entities) { entityIndexMap.put(s, JsonUtils.readValue(jsonPayload.get(s).toString(), IndexMapping.class)); } } catch (Exception e) { throw new RuntimeException("Failed to load indexMapping.json"); } + try (InputStream in2 = getClass().getResourceAsStream("/elasticsearch/collate/indexMapping.json")) { + if (in2 != null) { + JsonObject jsonPayload = JsonUtils.readJson(new String(in2.readAllBytes())).asJsonObject(); + entities = jsonPayload.keySet(); + for (String s : entities) { + entityIndexMap.put(s, JsonUtils.readValue(jsonPayload.get(s).toString(), IndexMapping.class)); + } + } + } catch (Exception e) { + LOG.warn("Failed to load indexMapping.json"); + } } public ElasticSearchConfiguration.SearchType getSearchType() { @@ -202,7 +218,7 @@ public class SearchRepository { String entityType = entity.getEntityReference().getType(); try { IndexMapping indexMapping = entityIndexMap.get(entityType); - SearchIndex index = SearchIndexFactory.buildIndex(entityType, entity); + SearchIndex index = searchIndexFactory.buildIndex(entityType, entity); String doc = JsonUtils.pojoToJson(index.buildESDoc()); searchClient.createEntity(indexMapping.getIndexName(), entityId, doc); } catch (Exception ie) { @@ -226,7 +242,7 @@ public class SearchRepository { String entityId = entity.getId().toString(); try { IndexMapping indexMapping = entityIndexMap.get(entityType); - SearchIndex index = SearchIndexFactory.buildIndex(entityType, entity); + SearchIndex index = searchIndexFactory.buildIndex(entityType, entity); String doc = JsonUtils.pojoToJson(index.buildESDoc()); searchClient.createTimeSeriesEntity(indexMapping.getIndexName(), entityId, doc); } catch (Exception ie) { @@ -250,7 +266,7 @@ public class SearchRepository { && Objects.equals(entity.getVersion(), entity.getChangeDescription().getPreviousVersion())) { scriptTxt = getScriptWithParams(entity, doc); } else { - SearchIndex elasticSearchIndex = SearchIndexFactory.buildIndex(entityType, entity); + SearchIndex elasticSearchIndex = searchIndexFactory.buildIndex(entityType, entity); doc = elasticSearchIndex.buildESDoc(); } searchClient.updateEntity(indexMapping.getIndexName(), entityId, doc, scriptTxt); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 627d7dd63ac..3d32de9c651 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -905,6 +905,7 @@ public class ElasticSearchClient implements SearchClient { AggregationBuilders.terms(OWNER_DISPLAY_NAME_KEYWORD) .field(OWNER_DISPLAY_NAME_KEYWORD) .size(MAX_AGGREGATE_SIZE)) + .aggregation(AggregationBuilders.terms("owner.displayName").field("owner.displayName").size(MAX_AGGREGATE_SIZE)) .aggregation( AggregationBuilders.terms(DOMAIN_DISPLAY_NAME_KEYWORD) .field(DOMAIN_DISPLAY_NAME_KEYWORD) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java index 7e2fd7eef22..8e82098739e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java @@ -15,7 +15,6 @@ import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.system.StepStats; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.ProcessorException; -import org.openmetadata.service.search.SearchIndexFactory; import org.openmetadata.service.search.models.IndexMapping; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.ResultList; @@ -73,7 +72,7 @@ public class ElasticSearchEntitiesProcessor implements Processor