From d906bcfcacc67356f9d81af6b892c46ea234e5b5 Mon Sep 17 00:00:00 2001 From: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com> Date: Tue, 20 Aug 2024 23:14:30 +0530 Subject: [PATCH] Adds Ability to Extend search functionalty (#17508) * Adds Ability to Extend search functionalty * Fix Tests --- .../java/org/openmetadata/service/Entity.java | 4 ++ .../service/OpenMetadataApplication.java | 12 ++++-- .../service/resources/apps/AppResource.java | 2 +- .../service/search/SearchClient.java | 5 +++ .../service/search/SearchRepository.java | 39 ++++++++++--------- .../elasticsearch/ElasticSearchClient.java | 2 +- .../search/opensearch/OpenSearchClient.java | 2 +- .../service/OpenMetadataApplicationTest.java | 4 +- 8 files changed, 45 insertions(+), 25 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 8d972a371b1..27328568c1d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -596,4 +596,8 @@ public final class Entity { public static T getDao() { return (T) collectionDAO; } + + public static T getSearchRepo() { + return (T) searchRepository; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index 0b433bdf5c7..8cf9242a8d4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -65,6 +65,7 @@ import org.openmetadata.schema.api.security.AuthenticationConfiguration; import org.openmetadata.schema.api.security.AuthorizerConfiguration; import org.openmetadata.schema.api.security.ClientType; import org.openmetadata.schema.configuration.LimitsConfiguration; +import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.services.connections.metadata.AuthProvider; import org.openmetadata.service.apps.ApplicationHandler; import org.openmetadata.service.apps.scheduler.AppScheduler; @@ -164,9 +165,7 @@ public class OpenMetadataApplication extends Application { // Create an On Demand DAO CollectionDAO dao = Entity.getCollectionDAO(); - searchRepository = new SearchRepository(config.getElasticSearchConfiguration()); + searchRepository = Entity.getSearchRepository(); AppScheduler.initialize(config, dao, searchRepository); // Initialize Default Apps diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index 982439296d7..25c993edc70 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -135,6 +135,11 @@ public interface SearchClient { String entityType) throws IOException; + default Response listPageHierarchy(String parent) { + throw new CustomExceptionMessage( + Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_ERROR_TYPE, NOT_IMPLEMENTED_METHOD); + } + Map searchLineageInternal( String fqn, int upstreamDepth, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index b7a9433149b..ae3235b82a0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -109,30 +109,14 @@ public class SearchRepository { public SearchRepository(ElasticSearchConfiguration config) { elasticSearchConfiguration = config; - if (config != null - && config.getSearchType() == ElasticSearchConfiguration.SearchType.OPENSEARCH) { - searchClient = new OpenSearchClient(config); - } else { - searchClient = new ElasticSearchClient(config); - } - try { - if (config != null && (!nullOrEmpty(config.getSearchIndexFactoryClassName()))) { - this.searchIndexFactory = - Class.forName(config.getSearchIndexFactoryClassName()) - .asSubclass(SearchIndexFactory.class) - .getDeclaredConstructor() - .newInstance(); - } - } catch (Exception e) { - LOG.warn("Failed to initialize search index factory using default one", e); - } + searchClient = buildSearchClient(config); + searchIndexFactory = buildIndexFactory(); language = config != null && config.getSearchIndexMappingLanguage() != null ? config.getSearchIndexMappingLanguage().value() : "en"; clusterAlias = config != null ? config.getClusterAlias() : ""; loadIndexMappings(); - Entity.setSearchRepository(this); } private void loadIndexMappings() { @@ -164,6 +148,21 @@ public class SearchRepository { } } + public SearchClient buildSearchClient(ElasticSearchConfiguration config) { + SearchClient sc; + if (config != null + && config.getSearchType() == ElasticSearchConfiguration.SearchType.OPENSEARCH) { + sc = new OpenSearchClient(config); + } else { + sc = new ElasticSearchClient(config); + } + return sc; + } + + public SearchIndexFactory buildIndexFactory() { + return new SearchIndexFactory(); + } + public ElasticSearchConfiguration.SearchType getSearchType() { return searchClient.getSearchType(); } @@ -881,4 +880,8 @@ public class SearchRepository { } return new ArrayList<>(); } + + public T getRestHighLevelClient() { + return (T) searchClient; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 7da93048fee..f3461028d0e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -208,7 +208,7 @@ import org.openmetadata.service.workflows.searchIndex.ReindexingUtil; public class ElasticSearchClient implements SearchClient { @SuppressWarnings("deprecated") - private final RestHighLevelClient client; + protected final RestHighLevelClient client; private final boolean isClientAvailable; public static final NamedXContentRegistry xContentRegistry; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 963e1cccf2d..8acb21de63d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -205,7 +205,7 @@ import os.org.opensearch.search.suggest.completion.context.CategoryQueryContext; @Slf4j // Not tagged with Repository annotation as it is programmatically initialized public class OpenSearchClient implements SearchClient { - private final RestHighLevelClient client; + protected final RestHighLevelClient client; public static final NamedXContentRegistry X_CONTENT_REGISTRY; private final boolean isClientAvailable; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java index fd5ed40283a..897ee0af519 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java @@ -232,7 +232,9 @@ public abstract class OpenMetadataApplicationTest { pipelineServiceClientConfiguration, forceMigrations); // Initialize search repository - new SearchRepository(config.getElasticSearchConfiguration()); + SearchRepository searchRepository = + new SearchRepository(config.getElasticSearchConfiguration()); + Entity.setSearchRepository(searchRepository); Entity.setCollectionDAO(jdbi.onDemand(CollectionDAO.class)); Entity.initializeRepositories(config, jdbi); workflow.loadMigrations();