yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-07 23:04:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #8496: Segregate ES indexes for use in shared infrastructure (#14606)

Browse Source 
* Fix #8496: Segregate ES indexes for use in shared infrastructure

* Fix tests

* use entityType aggregation key for counts

* localisation

* Revert openmetadata.yaml changes

* Update sample_data.yaml

---------

Co-authored-by: karanh37 <karanh37@gmail.com>
This commit is contained in:
Sriharsha Chintalapani 2024-01-08 08:29:11 -08:00 committed by GitHub
parent dcff230bb6
commit b9781746ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 351 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
conf/openmetadata.yaml
View File

@ -241,6 +241,7 @@ elasticsearch:
scheme: ${ELASTICSEARCH_SCHEME:-http}
username: ${ELASTICSEARCH_USER:-""}
password: ${ELASTICSEARCH_PASSWORD:-""}
clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-""}
truststorePath: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
truststorePassword: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
connectionTimeoutSecs: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}

3
openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertUtil.java
View File

@ -141,7 +141,8 @@ public final class AlertUtil {
for (String entityType : entities) {
try {
IndexMapping indexMapping = Entity.getSearchRepository().getIndexMapping(entityType);
indexesToSearch.add(indexMapping.getIndexName());
indexesToSearch.add(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
} catch (RuntimeException ex) {
LOG.error("Failing to get Index for EntityType");
}

3
openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java
View File

@ -158,7 +158,8 @@ public class SearchResource {
}
SearchRequest request =
new SearchRequest.ElasticSearchRequestBuilder(query, size, index)
new SearchRequest.ElasticSearchRequestBuilder(
query, size, Entity.getSearchRepository().getIndexOrAliasName(index))
.from(from)
.queryFilter(queryFilter)
.postFilter(postFilter)

73
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java
View File

@ -20,6 +20,7 @@ import static org.openmetadata.service.search.SearchClient.REMOVE_TEST_SUITE_CHI
import static org.openmetadata.service.search.SearchClient.SOFT_DELETE_RESTORE_SCRIPT;
import static org.openmetadata.service.search.SearchClient.UPDATE_ADDED_DELETE_GLOSSARY_TAGS;
import static org.openmetadata.service.search.SearchClient.UPDATE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT;
import static org.openmetadata.service.search.models.IndexMapping.indexNameSeparator;
import com.fasterxml.jackson.core.type.TypeReference;
import java.io.IOException;
@ -64,7 +65,7 @@ import org.openmetadata.service.util.JsonUtils;
@Slf4j
public class SearchRepository {
private final SearchClient searchClient;
@Getter private final SearchClient searchClient;
private Map<String, IndexMapping> entityIndexMap;
@ -78,6 +79,9 @@ public class SearchRepository {
@Getter private final ElasticSearchConfiguration elasticSearchConfiguration;
@Getter private final String clusterAlias;
@Getter
public final List<String> dataInsightReports =
List.of(
ENTITY_REPORT_DATA,
@ -98,19 +102,15 @@ public class SearchRepository {
searchClient = new ElasticSearchClient(config);
}
this.searchIndexFactory = searchIndexFactory;
this.language = config != null ? config.getSearchIndexMappingLanguage().value() : "en";
language =
config != null && config.getSearchIndexMappingLanguage() != null
? config.getSearchIndexMappingLanguage().value()
: "en";
clusterAlias = config != null ? config.getClusterAlias() : "";
loadIndexMappings();
Entity.setSearchRepository(this);
}
public SearchClient getSearchClient() {
return searchClient;
}
public List<String> getDataInsightReports() {
return dataInsightReports;
}
private void loadIndexMappings() {
Set<String> entities;
entityIndexMap = new HashMap<>();
@ -166,8 +166,14 @@ public class SearchRepository {
return entityIndexMap.get(entityType);
}
public String getIndexOrAliasName(String name) {
return clusterAlias != null && !clusterAlias.isEmpty()
? clusterAlias + indexNameSeparator + name
: name;
}
public boolean indexExists(IndexMapping indexMapping) {
return searchClient.indexExists(indexMapping.getIndexName());
return searchClient.indexExists(indexMapping.getIndexName(clusterAlias));
}
public void createIndex(IndexMapping indexMapping) {
@ -180,7 +186,8 @@ public class SearchRepository {
} catch (Exception e) {
LOG.error(
String.format(
"Failed to Create Index for entity %s due to ", indexMapping.getIndexName()),
"Failed to Create Index for entity %s due to ",
indexMapping.getIndexName(clusterAlias)),
e);
}
}
@ -195,7 +202,9 @@ public class SearchRepository {
}
searchClient.createAliases(indexMapping);
} catch (Exception e) {
LOG.warn(String.format("Failed to Update Index for entity %s", indexMapping.getIndexName()));
LOG.warn(
String.format(
"Failed to Update Index for entity %s", indexMapping.getIndexName(clusterAlias)));
}
}
@ -207,7 +216,8 @@ public class SearchRepository {
} catch (Exception e) {
LOG.error(
String.format(
"Failed to Delete Index for entity %s due to ", indexMapping.getIndexName()),
"Failed to Delete Index for entity %s due to ",
indexMapping.getIndexName(clusterAlias)),
e);
}
}
@ -233,7 +243,7 @@ public class SearchRepository {
IndexMapping indexMapping = entityIndexMap.get(entityType);
SearchIndex index = searchIndexFactory.buildIndex(entityType, entity);
String doc = JsonUtils.pojoToJson(index.buildESDoc());
searchClient.createEntity(indexMapping.getIndexName(), entityId, doc);
searchClient.createEntity(indexMapping.getIndexName(clusterAlias), entityId, doc);
} catch (Exception ie) {
LOG.error(
String.format(
@ -261,7 +271,7 @@ public class SearchRepository {
IndexMapping indexMapping = entityIndexMap.get(entityType);
SearchIndex index = searchIndexFactory.buildIndex(entityType, entity);
String doc = JsonUtils.pojoToJson(index.buildESDoc());
searchClient.createTimeSeriesEntity(indexMapping.getIndexName(), entityId, doc);
searchClient.createTimeSeriesEntity(indexMapping.getIndexName(clusterAlias), entityId, doc);
} catch (Exception ie) {
LOG.error(
String.format(
@ -291,7 +301,8 @@ public class SearchRepository {
SearchIndex elasticSearchIndex = searchIndexFactory.buildIndex(entityType, entity);
doc = elasticSearchIndex.buildESDoc();
}
searchClient.updateEntity(indexMapping.getIndexName(), entityId, doc, scriptTxt);
searchClient.updateEntity(
indexMapping.getIndexName(clusterAlias), entityId, doc, scriptTxt);
propagateInheritedFieldsToChildren(
entityType, entityId, entity.getChangeDescription(), indexMapping);
propagateGlossaryTags(
@ -339,7 +350,7 @@ public class SearchRepository {
parentMatch = new ImmutablePair<>(entityType + ".id", entityId);
}
if (updates.getKey() != null && !updates.getKey().isEmpty()) {
searchClient.updateChildren(indexMapping.getAlias(), parentMatch, updates);
searchClient.updateChildren(indexMapping.getAlias(clusterAlias), parentMatch, updates);
}
}
}
@ -440,7 +451,7 @@ public class SearchRepository {
public void deleteByScript(String entityType, String scriptTxt, Map<String, Object> params) {
try {
IndexMapping indexMapping = getIndexMapping(entityType);
searchClient.deleteByScript(indexMapping.getIndexName(), scriptTxt, params);
searchClient.deleteByScript(indexMapping.getIndexName(clusterAlias), scriptTxt, params);
} catch (Exception ie) {
LOG.error(
String.format(
@ -455,7 +466,7 @@ public class SearchRepository {
String entityType = entity.getEntityReference().getType();
IndexMapping indexMapping = entityIndexMap.get(entityType);
try {
searchClient.deleteEntity(indexMapping.getIndexName(), entityId);
searchClient.deleteEntity(indexMapping.getIndexName(clusterAlias), entityId);
deleteOrUpdateChildren(entity, indexMapping);
} catch (Exception ie) {
LOG.error(
@ -477,7 +488,8 @@ public class SearchRepository {
IndexMapping indexMapping = entityIndexMap.get(entityType);
String scriptTxt = String.format(SOFT_DELETE_RESTORE_SCRIPT, delete);
try {
searchClient.softDeleteOrRestoreEntity(indexMapping.getIndexName(), entityId, scriptTxt);
searchClient.softDeleteOrRestoreEntity(
indexMapping.getIndexName(clusterAlias), entityId, scriptTxt);
softDeleteOrRestoredChildren(entity, indexMapping, delete);
} catch (Exception ie) {
LOG.error(
@ -504,7 +516,8 @@ public class SearchRepository {
// we are doing below because we want to delete the data products with domain when domain is
// deleted
searchClient.deleteEntityByFields(
indexMapping.getAlias(), List.of(new ImmutablePair<>(entityType + ".id", docId)));
indexMapping.getAlias(clusterAlias),
List.of(new ImmutablePair<>(entityType + ".id", docId)));
}
case Entity.TAG, Entity.GLOSSARY_TERM -> searchClient.updateChildren(
GLOBAL_SEARCH_ALIAS,
@ -514,10 +527,11 @@ public class SearchRepository {
TestSuite testSuite = (TestSuite) entity;
if (Boolean.TRUE.equals(testSuite.getExecutable())) {
searchClient.deleteEntityByFields(
indexMapping.getAlias(), List.of(new ImmutablePair<>("testSuites.id", docId)));
indexMapping.getAlias(clusterAlias),
List.of(new ImmutablePair<>("testSuites.id", docId)));
} else {
searchClient.updateChildren(
indexMapping.getAlias(),
indexMapping.getAlias(clusterAlias),
new ImmutablePair<>("testSuites.id", testSuite.getId().toString()),
new ImmutablePair<>(REMOVE_TEST_SUITE_CHILDREN_SCRIPT, null));
}
@ -529,9 +543,10 @@ public class SearchRepository {
Entity.MLMODEL_SERVICE,
Entity.STORAGE_SERVICE,
Entity.SEARCH_SERVICE -> searchClient.deleteEntityByFields(
indexMapping.getAlias(), List.of(new ImmutablePair<>("service.id", docId)));
indexMapping.getAlias(clusterAlias), List.of(new ImmutablePair<>("service.id", docId)));
default -> searchClient.deleteEntityByFields(
indexMapping.getAlias(), List.of(new ImmutablePair<>(entityType + ".id", docId)));
indexMapping.getAlias(clusterAlias),
List.of(new ImmutablePair<>(entityType + ".id", docId)));
}
}
@ -548,9 +563,11 @@ public class SearchRepository {
Entity.MLMODEL_SERVICE,
Entity.STORAGE_SERVICE,
Entity.SEARCH_SERVICE -> searchClient.softDeleteOrRestoreChildren(
indexMapping.getAlias(), scriptTxt, List.of(new ImmutablePair<>("service.id", docId)));
indexMapping.getAlias(clusterAlias),
scriptTxt,
List.of(new ImmutablePair<>("service.id", docId)));
default -> searchClient.softDeleteOrRestoreChildren(
indexMapping.getAlias(),
indexMapping.getAlias(clusterAlias),
scriptTxt,
List.of(new ImmutablePair<>(entityType + ".id", docId)));
}

86
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -114,6 +114,7 @@ import org.apache.http.impl.client.BasicCredentialsProvider;
import org.openmetadata.schema.DataInsightInterface;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
import org.openmetadata.service.Entity;
import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;
import org.openmetadata.service.jdbi3.DataInsightChartRepository;
import org.openmetadata.service.search.SearchClient;
@ -165,13 +166,16 @@ public class ElasticSearchClient implements SearchClient {
private final boolean isClientAvailable;
private static final NamedXContentRegistry xContentRegistry;
private final String clusterAlias;
static {
SearchModule searchModule = new SearchModule(Settings.EMPTY, false, List.of());
xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
}
public ElasticSearchClient(ElasticSearchConfiguration esConfig) {
client = createElasticSearchClient(esConfig);
public ElasticSearchClient(ElasticSearchConfiguration config) {
client = createElasticSearchClient(config);
clusterAlias = config != null ? config.getClusterAlias() : "";
isClientAvailable = client != null;
}
@ -196,12 +200,15 @@ public class ElasticSearchClient implements SearchClient {
public void createIndex(IndexMapping indexMapping, String indexMappingContent) {
if (Boolean.TRUE.equals(isClientAvailable)) {
try {
CreateIndexRequest request = new CreateIndexRequest(indexMapping.getIndexName());
CreateIndexRequest request =
new CreateIndexRequest(indexMapping.getIndexName(clusterAlias));
request.source(indexMappingContent, XContentType.JSON);
CreateIndexResponse createIndexResponse =
client.indices().create(request, RequestOptions.DEFAULT);
LOG.debug(
"{} Created {}", indexMapping.getIndexName(), createIndexResponse.isAcknowledged());
"{} Created {}",
indexMapping.getIndexName(clusterAlias),
createIndexResponse.isAcknowledged());
// creating alias for indexes
createAliases(indexMapping);
} catch (Exception e) {
@ -216,25 +223,27 @@ public class ElasticSearchClient implements SearchClient {
@Override
public void createAliases(IndexMapping indexMapping) {
try {
Set<String> aliases = new HashSet<>(indexMapping.getParentAliases());
aliases.add(indexMapping.getAlias());
Set<String> aliases = new HashSet<>(indexMapping.getParentAliases(clusterAlias));
aliases.add(indexMapping.getAlias(clusterAlias));
IndicesAliasesRequest.AliasActions aliasAction =
IndicesAliasesRequest.AliasActions.add()
.index(indexMapping.getIndexName())
.index(indexMapping.getIndexName(clusterAlias))
.aliases(aliases.toArray(new String[0]));
IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
aliasesRequest.addAliasAction(aliasAction);
client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
} catch (Exception e) {
LOG.error(
String.format("Failed to create alias for %s due to", indexMapping.getIndexName()), e);
String.format(
"Failed to create alias for %s due to", indexMapping.getAlias(clusterAlias)),
e);
}
}
@Override
public void updateIndex(IndexMapping indexMapping, String indexMappingContent) {
try {
PutMappingRequest request = new PutMappingRequest(indexMapping.getIndexName());
PutMappingRequest request = new PutMappingRequest(indexMapping.getIndexName(clusterAlias));
JsonNode readProperties = JsonUtils.readTree(indexMappingContent).get("mappings");
request.source(JsonUtils.getMap(readProperties));
AcknowledgedResponse putMappingResponse =
@ -243,17 +252,21 @@ public class ElasticSearchClient implements SearchClient {
"{} Updated {}", indexMapping.getIndexMappingFile(), putMappingResponse.isAcknowledged());
} catch (Exception e) {
LOG.warn(
String.format("Failed to Update Elastic Search index %s", indexMapping.getIndexName()));
String.format(
"Failed to Update Elastic Search index %s", indexMapping.getIndexName(clusterAlias)));
}
}
@Override
public void deleteIndex(IndexMapping indexMapping) {
try {
DeleteIndexRequest request = new DeleteIndexRequest(indexMapping.getIndexName());
DeleteIndexRequest request = new DeleteIndexRequest(indexMapping.getIndexName(clusterAlias));
AcknowledgedResponse deleteIndexResponse =
client.indices().delete(request, RequestOptions.DEFAULT);
LOG.debug("{} Deleted {}", indexMapping.getIndexName(), deleteIndexResponse.isAcknowledged());
LOG.debug(
"{} Deleted {}",
indexMapping.getIndexName(clusterAlias),
deleteIndexResponse.isAcknowledged());
} catch (IOException e) {
LOG.error("Failed to delete Elastic Search indexes due to", e);
}
@ -263,35 +276,42 @@ public class ElasticSearchClient implements SearchClient {
public Response search(SearchRequest request) throws IOException {
SearchSourceBuilder searchSourceBuilder =
switch (request.getIndex()) {
case "topic_search_index" -> buildTopicSearchBuilder(
case "topic_search_index", "topic" -> buildTopicSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "dashboard_search_index" -> buildDashboardSearchBuilder(
case "dashboard_search_index", "dashboard" -> buildDashboardSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "pipeline_search_index" -> buildPipelineSearchBuilder(
case "pipeline_search_index", "pipeline" -> buildPipelineSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "mlmodel_search_index" -> buildMlModelSearchBuilder(
case "mlmodel_search_index", "mlmodel" -> buildMlModelSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "table_search_index" -> buildTableSearchBuilder(
case "table_search_index", "table" -> buildTableSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "user_search_index", "team_search_index" -> buildUserOrTeamSearchBuilder(
case "user_search_index",
"user",
"team_search_index",
"team" -> buildUserOrTeamSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "glossary_term_search_index" -> buildGlossaryTermSearchBuilder(
case "glossary_term_search_index", "glossaryTerm" -> buildGlossaryTermSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "tag_search_index" -> buildTagSearchBuilder(
case "tag_search_index", "tag" -> buildTagSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "container_search_index" -> buildContainerSearchBuilder(
case "container_search_index", "container" -> buildContainerSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "query_search_index" -> buildQuerySearchBuilder(
case "query_search_index", "query" -> buildQuerySearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "test_case_search_index", "test_suite_search_index" -> buildTestCaseSearch(
case "test_case_search_index",
"testCase",
"test_suite_search_index",
"testSuite" -> buildTestCaseSearch(
request.getQuery(), request.getFrom(), request.getSize());
case "stored_procedure_search_index" -> buildStoredProcedureSearch(
case "stored_procedure_search_index", "storedProcedure" -> buildStoredProcedureSearch(
request.getQuery(), request.getFrom(), request.getSize());
case "dashboard_data_model_search_index" -> buildDashboardDataModelsSearch(
case "dashboard_data_model_search_index",
"dashboardDataModel" -> buildDashboardDataModelsSearch(
request.getQuery(), request.getFrom(), request.getSize());
case "search_entity_search_index" -> buildSearchEntitySearch(
case "search_entity_search_index", "searchIndex" -> buildSearchEntitySearch(
request.getQuery(), request.getFrom(), request.getSize());
case "domain_search_index" -> buildDomainsSearch(
case "domain_search_index", "domain" -> buildDomainsSearch(
request.getQuery(), request.getFrom(), request.getSize());
case "raw_cost_analysis_report_data_index",
"aggregated_cost_analysis_report_data_index" -> buildCostAnalysisReportDataSearch(
@ -402,7 +422,8 @@ public class ElasticSearchClient implements SearchClient {
@Override
public Response searchBySourceUrl(String sourceUrl) throws IOException {
es.org.elasticsearch.action.search.SearchRequest searchRequest =
new es.org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS);
new es.org.elasticsearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(
QueryBuilders.boolQuery().must(QueryBuilders.termQuery("sourceUrl", sourceUrl)));
@ -415,7 +436,8 @@ public class ElasticSearchClient implements SearchClient {
public Response searchByField(String fieldName, String fieldValue, String index)
throws IOException {
es.org.elasticsearch.action.search.SearchRequest searchRequest =
new es.org.elasticsearch.action.search.SearchRequest(index);
new es.org.elasticsearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(index));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.wildcardQuery(fieldName, fieldValue));
searchRequest.source(searchSourceBuilder);
@ -447,7 +469,8 @@ public class ElasticSearchClient implements SearchClient {
String response =
client
.search(
new es.org.elasticsearch.action.search.SearchRequest(index)
new es.org.elasticsearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(index))
.source(searchSourceBuilder),
RequestOptions.DEFAULT)
.toString();
@ -483,7 +506,8 @@ public class ElasticSearchClient implements SearchClient {
request.getIncludeSourceFields().toArray(String[]::new),
new String[] {}));
es.org.elasticsearch.action.search.SearchRequest searchRequest =
new es.org.elasticsearch.action.search.SearchRequest(request.getIndex())
new es.org.elasticsearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(request.getIndex()))
.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
Suggest suggest = searchResponse.getSuggest();

4
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java
View File

@ -76,7 +76,9 @@ public class ElasticSearchDataInsightProcessor
private UpdateRequest getUpdateRequest(String entityType, ReportData reportData) {
IndexMapping indexMapping = Entity.getSearchRepository().getIndexMapping(entityType);
UpdateRequest updateRequest =
new UpdateRequest(indexMapping.getIndexName(), reportData.getId().toString());
new UpdateRequest(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()),
reportData.getId().toString());
updateRequest.doc(
JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildESDoc()), XContentType.JSON);
updateRequest.docAsUpsert(true);

4
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java
View File

@ -78,7 +78,9 @@ public class ElasticSearchEntitiesProcessor
public static UpdateRequest getUpdateRequest(String entityType, EntityInterface entity) {
IndexMapping indexMapping = Entity.getSearchRepository().getIndexMapping(entityType);
UpdateRequest updateRequest =
new UpdateRequest(indexMapping.getIndexName(), entity.getId().toString());
new UpdateRequest(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()),
entity.getId().toString());
updateRequest.doc(
JsonUtils.pojoToJson(
Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity)).buildESDoc()),

31
openmetadata-service/src/main/java/org/openmetadata/service/search/models/IndexMapping.java
View File

@ -15,4 +15,35 @@ public class IndexMapping {
String indexMappingFile;
String alias;
List<String> parentAliases;
public static final String indexNameSeparator = "_";
public String getIndexName(String clusterAlias) {
return clusterAlias != null && !clusterAlias.isEmpty()
? clusterAlias + indexNameSeparator + indexName
: indexName;
}
public String getAlias(String clusterAlias) {
return clusterAlias != null && !clusterAlias.isEmpty()
? clusterAlias + indexNameSeparator + alias
: alias;
}
public List<String> getParentAliases(String clusterAlias) {
return clusterAlias != null && !clusterAlias.isEmpty()
? parentAliases.stream().map(alias -> clusterAlias + indexNameSeparator + alias).toList()
: parentAliases;
}
private String getIndexName() {
return indexName;
}
private String getAlias() {
return alias;
}
private List<String> getParentAliases() {
return parentAliases;
}
}

84
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -45,6 +45,7 @@ import org.apache.http.impl.client.BasicCredentialsProvider;
import org.openmetadata.schema.DataInsightInterface;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
import org.openmetadata.service.Entity;
import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;
import org.openmetadata.service.jdbi3.DataInsightChartRepository;
import org.openmetadata.service.search.SearchClient;
@ -158,13 +159,16 @@ public class OpenSearchClient implements SearchClient {
private static final NamedXContentRegistry X_CONTENT_REGISTRY;
private final boolean isClientAvailable;
private final String clusterAlias;
static {
SearchModule searchModule = new SearchModule(Settings.EMPTY, List.of());
X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents());
}
public OpenSearchClient(ElasticSearchConfiguration esConfig) {
client = createOpenSearchClient(esConfig);
public OpenSearchClient(ElasticSearchConfiguration config) {
client = createOpenSearchClient(config);
clusterAlias = config != null ? config.getClusterAlias() : "";
isClientAvailable = client != null;
}
@ -189,12 +193,15 @@ public class OpenSearchClient implements SearchClient {
public void createIndex(IndexMapping indexMapping, String indexMappingContent) {
if (Boolean.TRUE.equals(isClientAvailable)) {
try {
CreateIndexRequest request = new CreateIndexRequest(indexMapping.getIndexName());
CreateIndexRequest request =
new CreateIndexRequest(indexMapping.getIndexName(clusterAlias));
request.source(indexMappingContent, XContentType.JSON);
CreateIndexResponse createIndexResponse =
client.indices().create(request, RequestOptions.DEFAULT);
LOG.debug(
"{} Created {}", indexMapping.getIndexName(), createIndexResponse.isAcknowledged());
"{} Created {}",
indexMapping.getIndexName(clusterAlias),
createIndexResponse.isAcknowledged());
// creating alias for indexes
createAliases(indexMapping);
} catch (Exception e) {
@ -209,25 +216,27 @@ public class OpenSearchClient implements SearchClient {
@Override
public void createAliases(IndexMapping indexMapping) {
try {
List<String> aliases = indexMapping.getParentAliases();
aliases.add(indexMapping.getAlias());
List<String> aliases = indexMapping.getParentAliases(clusterAlias);
aliases.add(indexMapping.getAlias(clusterAlias));
IndicesAliasesRequest.AliasActions aliasAction =
IndicesAliasesRequest.AliasActions.add()
.index(indexMapping.getIndexName())
.index(indexMapping.getIndexName(clusterAlias))
.aliases(aliases.toArray(new String[0]));
IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
aliasesRequest.addAliasAction(aliasAction);
client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
} catch (Exception e) {
LOG.error(
String.format("Failed to create alias for %s due to", indexMapping.getIndexName()), e);
String.format(
"Failed to create alias for %s due to", indexMapping.getIndexName(clusterAlias)),
e);
}
}
@Override
public void updateIndex(IndexMapping indexMapping, String indexMappingContent) {
try {
PutMappingRequest request = new PutMappingRequest(indexMapping.getIndexName());
PutMappingRequest request = new PutMappingRequest(indexMapping.getIndexName(clusterAlias));
JsonNode readProperties = JsonUtils.readTree(indexMappingContent).get("mappings");
request.source(JsonUtils.getMap(readProperties));
AcknowledgedResponse putMappingResponse =
@ -235,17 +244,22 @@ public class OpenSearchClient implements SearchClient {
LOG.debug(
"{} Updated {}", indexMapping.getIndexMappingFile(), putMappingResponse.isAcknowledged());
} catch (Exception e) {
LOG.warn(String.format("Failed to Update Open Search index %s", indexMapping.getIndexName()));
LOG.warn(
String.format(
"Failed to Update Open Search index %s", indexMapping.getIndexName(clusterAlias)));
}
}
@Override
public void deleteIndex(IndexMapping indexMapping) {
try {
DeleteIndexRequest request = new DeleteIndexRequest(indexMapping.getIndexName());
DeleteIndexRequest request = new DeleteIndexRequest(indexMapping.getIndexName(clusterAlias));
AcknowledgedResponse deleteIndexResponse =
client.indices().delete(request, RequestOptions.DEFAULT);
LOG.debug("{} Deleted {}", indexMapping.getIndexName(), deleteIndexResponse.isAcknowledged());
LOG.debug(
"{} Deleted {}",
indexMapping.getIndexName(clusterAlias),
deleteIndexResponse.isAcknowledged());
} catch (IOException e) {
LOG.error("Failed to delete Open Search indexes due to", e);
}
@ -255,35 +269,39 @@ public class OpenSearchClient implements SearchClient {
public Response search(SearchRequest request) throws IOException {
SearchSourceBuilder searchSourceBuilder =
switch (request.getIndex()) {
case "topic_search_index" -> buildTopicSearchBuilder(
case "topic_search_index", "topic" -> buildTopicSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "dashboard_search_index" -> buildDashboardSearchBuilder(
case "dashboard_search_index", "dashboard" -> buildDashboardSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "pipeline_search_index" -> buildPipelineSearchBuilder(
case "pipeline_search_index", "pipeline" -> buildPipelineSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "mlmodel_search_index" -> buildMlModelSearchBuilder(
case "mlmodel_search_index", "mlmodel" -> buildMlModelSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "table_search_index" -> buildTableSearchBuilder(
case "table_search_index", "table" -> buildTableSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "user_search_index", "team_search_index" -> buildUserOrTeamSearchBuilder(
case "user_search_index",
"user",
"team_search_index",
"team" -> buildUserOrTeamSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "glossary_term_search_index" -> buildGlossaryTermSearchBuilder(
case "glossary_term_search_index", "glossaryTerm" -> buildGlossaryTermSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "tag_search_index" -> buildTagSearchBuilder(
case "tag_search_index", "tag" -> buildTagSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "container_search_index" -> buildContainerSearchBuilder(
case "container_search_index", "container" -> buildContainerSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "query_search_index" -> buildQuerySearchBuilder(
case "query_search_index", "query" -> buildQuerySearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "test_case_search_index" -> buildTestCaseSearch(
case "test_case_search_index", "testCase" -> buildTestCaseSearch(
request.getQuery(), request.getFrom(), request.getSize());
case "stored_procedure_search_index" -> buildStoredProcedureSearch(
case "stored_procedure_search_index", "storedProcedure" -> buildStoredProcedureSearch(
request.getQuery(), request.getFrom(), request.getSize());
case "dashboard_data_model_search_index" -> buildDashboardDataModelsSearch(
case "dashboard_data_model_search_index",
"dashboardDataModel" -> buildDashboardDataModelsSearch(
request.getQuery(), request.getFrom(), request.getSize());
case "domain_search_index" -> buildDomainsSearch(
case "domain_search_index", "domain" -> buildDomainsSearch(
request.getQuery(), request.getFrom(), request.getSize());
case "search_entity_search_index" -> buildSearchEntitySearch(
case "search_entity_search_index", "searchIndex" -> buildSearchEntitySearch(
request.getQuery(), request.getFrom(), request.getSize());
case "raw_cost_analysis_report_data_index",
"aggregated_cost_analysis_report_data_index" -> buildCostAnalysisReportDataSearch(
@ -398,7 +416,8 @@ public class OpenSearchClient implements SearchClient {
@Override
public Response searchBySourceUrl(String sourceUrl) throws IOException {
os.org.opensearch.action.search.SearchRequest searchRequest =
new os.org.opensearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS);
new os.org.opensearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(
QueryBuilders.boolQuery().must(QueryBuilders.termQuery("sourceUrl", sourceUrl)));
@ -411,7 +430,8 @@ public class OpenSearchClient implements SearchClient {
public Response searchByField(String fieldName, String fieldValue, String index)
throws IOException {
os.org.opensearch.action.search.SearchRequest searchRequest =
new os.org.opensearch.action.search.SearchRequest(index);
new os.org.opensearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(index));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.wildcardQuery(fieldName, fieldValue));
searchRequest.source(searchSourceBuilder);
@ -442,7 +462,8 @@ public class OpenSearchClient implements SearchClient {
String response =
client
.search(
new os.org.opensearch.action.search.SearchRequest(index)
new os.org.opensearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(index))
.source(searchSourceBuilder),
RequestOptions.DEFAULT)
.toString();
@ -498,7 +519,8 @@ public class OpenSearchClient implements SearchClient {
request.getIncludeSourceFields().toArray(String[]::new),
new String[] {}));
os.org.opensearch.action.search.SearchRequest searchRequest =
new os.org.opensearch.action.search.SearchRequest(request.getIndex())
new os.org.opensearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(request.getIndex()))
.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
Suggest suggest = searchResponse.getSuggest();

4
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java
View File

@ -76,7 +76,9 @@ public class OpenSearchDataInsightProcessor
private UpdateRequest getUpdateRequest(String entityType, ReportData reportData) {
IndexMapping indexMapping = Entity.getSearchRepository().getIndexMapping(entityType);
UpdateRequest updateRequest =
new UpdateRequest(indexMapping.getIndexName(), reportData.getId().toString());
new UpdateRequest(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()),
reportData.getId().toString());
updateRequest.doc(
JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildESDoc()), XContentType.JSON);
updateRequest.docAsUpsert(true);

4
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntitiesProcessor.java
View File

@ -78,7 +78,9 @@ public class OpenSearchEntitiesProcessor
public static UpdateRequest getUpdateRequest(String entityType, EntityInterface entity) {
IndexMapping indexMapping = Entity.getSearchRepository().getIndexMapping(entityType);
UpdateRequest updateRequest =
new UpdateRequest(indexMapping.getIndexName(), entity.getId().toString());
new UpdateRequest(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()),
entity.getId().toString());
updateRequest.doc(
JsonUtils.pojoToJson(
Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity)).buildESDoc()),

24
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -1901,7 +1901,9 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
IndexMapping indexMapping =
Entity.getSearchRepository().getIndexMapping(entityReference.getType());
Awaitility.await().wait(2000L);
SearchResponse response = getResponseFormSearch(indexMapping.getIndexName());
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
List<String> entityIds = new ArrayList<>();
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
@ -1923,7 +1925,9 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
IndexMapping indexMapping =
Entity.getSearchRepository().getIndexMapping(entityReference.getType());
Awaitility.await().wait(2000L);
SearchResponse response = getResponseFormSearch(indexMapping.getIndexName());
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
List<String> entityIds = new ArrayList<>();
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
@ -1939,7 +1943,9 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
// search again in search after deleting
Awaitility.await().wait(2000L);
response = getResponseFormSearch(indexMapping.getIndexName());
response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
@ -1963,7 +1969,9 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
entity.setDescription("update description");
entity = patchEntity(entity.getId(), original, entity, ADMIN_AUTH_HEADERS);
Awaitility.await().wait(2000L);
SearchResponse response = getResponseFormSearch(indexMapping.getIndexName());
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
@ -1997,7 +2005,9 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
// add tags to entity
entity = patchEntity(entity.getId(), origJson, entity, ADMIN_AUTH_HEADERS);
Awaitility.await().wait(2000L);
SearchResponse response = getResponseFormSearch(indexMapping.getIndexName());
SearchResponse response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
@ -2014,7 +2024,9 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
// delete the tag
tagResourceTest.deleteEntity(tag.getId(), false, true, ADMIN_AUTH_HEADERS);
Awaitility.await().wait(2000L);
response = getResponseFormSearch(indexMapping.getIndexName());
response =
getResponseFormSearch(
indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()));
hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();

5
openmetadata-spec/src/main/resources/json/schema/configuration/elasticSearchConfiguration.json
View File

@ -66,6 +66,11 @@
"type": "integer",
"default": 10
},
"clusterAlias": {
"description": "Alias for search indexes to provide segregation of indexes.",
"type": "string",
"default": null
},
"searchIndexMappingLanguage": {
"$ref": "#/definitions/searchIndexMappingLanguage"
},

16
openmetadata-ui/src/main/resources/ui/src/constants/explore.constants.ts
View File

@ -12,7 +12,6 @@
*/
import { SortingField } from '../components/Explore/SortingDropDown';
import { SearchIndex } from '../enums/search.enum';
import i18n from '../utils/i18next/LocalUtil';
export const INITIAL_SORT_FIELD = 'updatedAt';
@ -58,18 +57,3 @@ export const COMMON_FILTERS_FOR_DIFFERENT_TABS = [
'owner.displayName',
'tags.tagFQN',
];
export const TABS_SEARCH_INDEXES = [
SearchIndex.TABLE,
SearchIndex.STORED_PROCEDURE,
SearchIndex.DASHBOARD,
SearchIndex.DASHBOARD_DATA_MODEL,
SearchIndex.PIPELINE,
SearchIndex.TOPIC,
SearchIndex.MLMODEL,
SearchIndex.CONTAINER,
SearchIndex.SEARCH_INDEX,
SearchIndex.GLOSSARY,
SearchIndex.TAG,
SearchIndex.DATA_PRODUCT,
];

19
openmetadata-ui/src/main/resources/ui/src/pages/ExplorePage/ExplorePageV1.component.tsx
View File

@ -37,18 +37,21 @@ import { getExplorePath, PAGE_SIZE } from '../../constants/constants';
import {
COMMON_FILTERS_FOR_DIFFERENT_TABS,
INITIAL_SORT_FIELD,
TABS_SEARCH_INDEXES,
} from '../../constants/explore.constants';
import {
mockSearchData,
MOCK_EXPLORE_PAGE_COUNT,
} from '../../constants/mockTourData.constants';
import { SORT_ORDER } from '../../enums/common.enum';
import { EntityType } from '../../enums/entity.enum';
import { SearchIndex } from '../../enums/search.enum';
import { Aggregations, SearchResponse } from '../../interface/search.interface';
import { searchQuery } from '../../rest/searchAPI';
import { getCountBadge } from '../../utils/CommonUtils';
import { findActiveSearchIndex } from '../../utils/Explore.utils';
import {
findActiveSearchIndex,
getSearchIndexFromEntityType,
} from '../../utils/Explore.utils';
import { getCombinedQueryFilterObject } from '../../utils/ExplorePage/ExplorePageUtils';
import searchClassBase from '../../utils/SearchClassBase';
import { escapeESReservedCharacters } from '../../utils/StringsUtils';
@ -64,6 +67,7 @@ const ExplorePageV1: FunctionComponent = () => {
const location = useLocation();
const history = useHistory();
const { isTourOpen } = useTourProvider();
const TABS_SEARCH_INDEXES = Object.keys(tabsInfo) as ExploreSearchIndex[];
const { tab } = useParams<UrlParams>();
@ -339,12 +343,17 @@ const ExplorePageV1: FunctionComponent = () => {
fetchSource: false,
filters: '',
}).then((res) => {
const buckets = res.aggregations[`index_count`].buckets;
const buckets = res.aggregations['entityType'].buckets;
const counts: Record<string, number> = {};
buckets.forEach((item) => {
if (item && TABS_SEARCH_INDEXES.includes(item.key as SearchIndex)) {
counts[item.key ?? ''] = item.doc_count;
const searchIndexKey =
item && getSearchIndexFromEntityType(item.key as EntityType);
if (
TABS_SEARCH_INDEXES.includes(searchIndexKey as ExploreSearchIndex)
) {
counts[searchIndexKey ?? ''] = item.doc_count;
}
});
setSearchHitCounts(counts as SearchHitCounts);

32
openmetadata-ui/src/main/resources/ui/src/utils/Assets/AssetsUtils.ts
View File

@ -16,7 +16,6 @@ import { EntityDetailUnion } from 'Models';
import { MapPatchAPIResponse } from '../../components/Assets/AssetsSelectionModal/AssetSelectionModal.interface';
import { AssetsOfEntity } from '../../components/Glossary/GlossaryTerms/tabs/AssetsTabs.interface';
import { EntityType } from '../../enums/entity.enum';
import { SearchIndex } from '../../enums/search.enum';
import { Table } from '../../generated/entity/data/table';
import { Domain } from '../../generated/entity/domains/domain';
import {
@ -163,37 +162,6 @@ export const getEntityAPIfromSource = (
}
};
export const getAssetsSearchIndex = (source: AssetsOfEntity) => {
const commonAssets: Record<string, SearchIndex> = {
[EntityType.ALL]: SearchIndex.ALL,
[EntityType.TABLE]: SearchIndex.TABLE,
[EntityType.PIPELINE]: SearchIndex.PIPELINE,
[EntityType.DASHBOARD]: SearchIndex.DASHBOARD,
[EntityType.MLMODEL]: SearchIndex.MLMODEL,
[EntityType.TOPIC]: SearchIndex.TOPIC,
[EntityType.CONTAINER]: SearchIndex.CONTAINER,
[EntityType.STORED_PROCEDURE]: SearchIndex.STORED_PROCEDURE,
[EntityType.DASHBOARD_DATA_MODEL]: SearchIndex.DASHBOARD_DATA_MODEL,
[EntityType.SEARCH_INDEX]: SearchIndex.SEARCH_INDEX,
[EntityType.DATABASE_SERVICE]: SearchIndex.DATABASE_SERVICE,
[EntityType.MESSAGING_SERVICE]: SearchIndex.MESSAGING_SERVICE,
[EntityType.DASHBOARD_SERVICE]: SearchIndex.DASHBOARD_SERVICE,
[EntityType.PIPELINE_SERVICE]: SearchIndex.PIPELINE_SERVICE,
[EntityType.MLMODEL_SERVICE]: SearchIndex.ML_MODEL_SERVICE,
[EntityType.STORAGE_SERVICE]: SearchIndex.STORAGE_SERVICE,
[EntityType.SEARCH_SERVICE]: SearchIndex.SEARCH_SERVICE,
};
if (
source === AssetsOfEntity.DOMAIN ||
source === AssetsOfEntity.DATA_PRODUCT
) {
commonAssets[EntityType.GLOSSARY] = SearchIndex.GLOSSARY;
}
return commonAssets;
};
export const getAssetsFields = (source: AssetsOfEntity) => {
if (source === AssetsOfEntity.GLOSSARY) {
return 'tags';

83
openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.test.ts Normal file
View File

@ -0,0 +1,83 @@
/*
* Copyright 2024 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityType } from '../enums/entity.enum';
import { SearchIndex } from '../enums/search.enum';
import { getSearchIndexFromEntityType } from './Explore.utils';
describe('Explore Utils', () => {
it('getSearchIndexFromEntityType should return the correct search index for each entity type', () => {
expect(getSearchIndexFromEntityType(EntityType.ALL)).toEqual(
SearchIndex.ALL
);
expect(getSearchIndexFromEntityType(EntityType.TABLE)).toEqual(
SearchIndex.TABLE
);
expect(getSearchIndexFromEntityType(EntityType.PIPELINE)).toEqual(
SearchIndex.PIPELINE
);
expect(getSearchIndexFromEntityType(EntityType.DASHBOARD)).toEqual(
SearchIndex.DASHBOARD
);
expect(getSearchIndexFromEntityType(EntityType.MLMODEL)).toEqual(
SearchIndex.MLMODEL
);
expect(getSearchIndexFromEntityType(EntityType.TOPIC)).toEqual(
SearchIndex.TOPIC
);
expect(getSearchIndexFromEntityType(EntityType.CONTAINER)).toEqual(
SearchIndex.CONTAINER
);
expect(getSearchIndexFromEntityType(EntityType.TAG)).toEqual(
SearchIndex.TAG
);
expect(getSearchIndexFromEntityType(EntityType.GLOSSARY_TERM)).toEqual(
SearchIndex.GLOSSARY
);
expect(getSearchIndexFromEntityType(EntityType.STORED_PROCEDURE)).toEqual(
SearchIndex.STORED_PROCEDURE
);
expect(
getSearchIndexFromEntityType(EntityType.DASHBOARD_DATA_MODEL)
).toEqual(SearchIndex.DASHBOARD_DATA_MODEL);
expect(getSearchIndexFromEntityType(EntityType.SEARCH_INDEX)).toEqual(
SearchIndex.SEARCH_INDEX
);
expect(getSearchIndexFromEntityType(EntityType.DATABASE_SERVICE)).toEqual(
SearchIndex.DATABASE_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.MESSAGING_SERVICE)).toEqual(
SearchIndex.MESSAGING_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.DASHBOARD_SERVICE)).toEqual(
SearchIndex.DASHBOARD_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.PIPELINE_SERVICE)).toEqual(
SearchIndex.PIPELINE_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.MLMODEL_SERVICE)).toEqual(
SearchIndex.ML_MODEL_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.STORAGE_SERVICE)).toEqual(
SearchIndex.STORAGE_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.SEARCH_SERVICE)).toEqual(
SearchIndex.SEARCH_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.DOMAIN)).toEqual(
SearchIndex.DOMAIN
);
expect(getSearchIndexFromEntityType(EntityType.DATA_PRODUCT)).toEqual(
SearchIndex.DATA_PRODUCT
);
});
});

30
openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts
View File

@ -18,6 +18,8 @@ import {
SearchHitCounts,
} from '../components/Explore/ExplorePage.interface';
import { SearchDropdownOption } from '../components/SearchDropdown/SearchDropdown.interface';
import { EntityType } from '../enums/entity.enum';
import { SearchIndex } from '../enums/search.enum';
import { Aggregations } from '../interface/search.interface';
import {
QueryFieldInterface,
@ -113,3 +115,31 @@ export const getAggregations = (data: Aggregations) => {
])
) as Aggregations;
};
export const getSearchIndexFromEntityType = (entityType: EntityType) => {
const commonAssets: Record<string, SearchIndex> = {
[EntityType.ALL]: SearchIndex.ALL,
[EntityType.TABLE]: SearchIndex.TABLE,
[EntityType.PIPELINE]: SearchIndex.PIPELINE,
[EntityType.DASHBOARD]: SearchIndex.DASHBOARD,
[EntityType.MLMODEL]: SearchIndex.MLMODEL,
[EntityType.TOPIC]: SearchIndex.TOPIC,
[EntityType.CONTAINER]: SearchIndex.CONTAINER,
[EntityType.TAG]: SearchIndex.TAG,
[EntityType.GLOSSARY_TERM]: SearchIndex.GLOSSARY,
[EntityType.STORED_PROCEDURE]: SearchIndex.STORED_PROCEDURE,
[EntityType.DASHBOARD_DATA_MODEL]: SearchIndex.DASHBOARD_DATA_MODEL,
[EntityType.SEARCH_INDEX]: SearchIndex.SEARCH_INDEX,
[EntityType.DATABASE_SERVICE]: SearchIndex.DATABASE_SERVICE,
[EntityType.MESSAGING_SERVICE]: SearchIndex.MESSAGING_SERVICE,
[EntityType.DASHBOARD_SERVICE]: SearchIndex.DASHBOARD_SERVICE,
[EntityType.PIPELINE_SERVICE]: SearchIndex.PIPELINE_SERVICE,
[EntityType.MLMODEL_SERVICE]: SearchIndex.ML_MODEL_SERVICE,
[EntityType.STORAGE_SERVICE]: SearchIndex.STORAGE_SERVICE,
[EntityType.SEARCH_SERVICE]: SearchIndex.SEARCH_SERVICE,
[EntityType.DOMAIN]: SearchIndex.DOMAIN,
[EntityType.DATA_PRODUCT]: SearchIndex.DATA_PRODUCT,
};
return commonAssets[entityType];
};