diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 9bca6d8d450..c8a8cb50ffc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -136,6 +136,7 @@ import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.Elas import org.openmetadata.service.search.indexes.ContainerIndex; import org.openmetadata.service.search.indexes.DashboardDataModelIndex; import org.openmetadata.service.search.indexes.DashboardIndex; +import org.openmetadata.service.search.indexes.DataProductIndex; import org.openmetadata.service.search.indexes.DomainIndex; import org.openmetadata.service.search.indexes.GlossaryTermIndex; import org.openmetadata.service.search.indexes.MlModelIndex; @@ -306,6 +307,9 @@ public class ElasticSearchClient implements SearchClient { searchSourceBuilder = buildCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize()); break; + case "data_product_search_index": + searchSourceBuilder = buildDataProductSearch(request.getQuery(), request.getFrom(), request.getSize()); + break; default: searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize()); break; @@ -339,7 +343,7 @@ public class ElasticSearchClient implements SearchClient { /* For backward-compatibility we continue supporting the deleted argument, this should be removed in future versions */ if (request.getIndex().equalsIgnoreCase("domain_search_index") - || request.getIndex().equalsIgnoreCase("data_products_search_index") + || request.getIndex().equalsIgnoreCase("data_product_search_index") || request.getIndex().equalsIgnoreCase("query_search_index") || request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index") || request.getIndex().equalsIgnoreCase("aggregated_cost_analysis_report_data_index")) { @@ -855,6 +859,33 @@ public class ElasticSearchClient implements SearchClient { return addAggregation(searchSourceBuilder); } + private static SearchSourceBuilder buildDataProductSearch(String query, int from, int size) { + QueryStringQueryBuilder queryBuilder = + QueryBuilders.queryStringQuery(query) + .fields(DataProductIndex.getFields()) + .defaultOperator(Operator.AND) + .fuzziness(Fuzziness.AUTO); + + HighlightBuilder hb = new HighlightBuilder(); + HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION); + highlightDescription.highlighterType(UNIFIED); + HighlightBuilder.Field highlightName = new HighlightBuilder.Field(FIELD_NAME); + highlightName.highlighterType(UNIFIED); + HighlightBuilder.Field highlightDisplayName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME); + highlightDisplayName.highlighterType(UNIFIED); + + hb.field(highlightDescription); + hb.field(highlightName); + hb.field(highlightDisplayName); + + hb.preTags(PRE_TAG); + hb.postTags(POST_TAG); + + SearchSourceBuilder searchSourceBuilder = + new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); + return addAggregation(searchSourceBuilder); + } + private static SearchSourceBuilder buildAggregateSearchBuilder(String query, int from, int size) { QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query).lenient(true); SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java index 527c56ed13b..d1852ebb9e1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java @@ -71,6 +71,7 @@ public class DashboardDataModelIndex implements ColumnIndex { getFQNParts( dashboardDataModel.getFullyQualifiedName(), suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList()))); + doc.put("tier", parseTags.getTierTag()); if (dashboardDataModel.getOwner() != null) { doc.put("owner", getOwnerWithDisplayName(dashboardDataModel.getOwner())); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java index cfcadd8c823..09bba3dce0f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java @@ -1,6 +1,16 @@ package org.openmetadata.service.search.indexes; +import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; +import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; +import static org.openmetadata.service.Entity.FIELD_NAME; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; + import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -40,4 +50,17 @@ public class DataProductIndex implements SearchIndex { } return doc; } + + public static Map getFields() { + Map fields = new HashMap<>(); + fields.put(FIELD_DISPLAY_NAME, 15.0f); + fields.put(FIELD_DISPLAY_NAME_NGRAM, 1.0f); + fields.put(FIELD_NAME, 15.0f); + fields.put(FIELD_NAME_NGRAM, 1.0f); + fields.put(DISPLAY_NAME_KEYWORD, 25.0f); + fields.put(NAME_KEYWORD, 25.0f); + fields.put(FULLY_QUALIFIED_NAME_PARTS, 10.0f); + fields.put(FIELD_DESCRIPTION, 1.0f); + return fields; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java index 53c996ab5a5..af69a3e9e76 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.openmetadata.service.Entity; +import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; import org.openmetadata.service.util.JsonUtils; @@ -42,6 +43,9 @@ public class SearchEntityIndex implements SearchIndex { getFQNParts( searchIndex.getFullyQualifiedName(), suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList()))); + ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.SEARCH_INDEX, searchIndex)); + doc.put("tags", parseTags.getTags()); + doc.put("tier", parseTags.getTierTag()); if (searchIndex.getOwner() != null) { doc.put("owner", getOwnerWithDisplayName(searchIndex.getOwner())); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java index daf5581e75c..33241026cba 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java @@ -14,6 +14,7 @@ import java.util.Map; import java.util.stream.Collectors; import org.openmetadata.schema.entity.data.StoredProcedure; import org.openmetadata.service.Entity; +import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; import org.openmetadata.service.util.JsonUtils; @@ -41,6 +42,9 @@ public class StoredProcedureIndex implements SearchIndex { suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList()))); doc.put("suggest", suggest); doc.put("entityType", Entity.STORED_PROCEDURE); + ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.STORED_PROCEDURE, storedProcedure)); + doc.put("tags", parseTags.getTags()); + doc.put("tier", parseTags.getTierTag()); if (storedProcedure.getOwner() != null) { doc.put("owner", getOwnerWithDisplayName(storedProcedure.getOwner())); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 71e4ae5dbac..dffa2b1b1f0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -51,6 +51,7 @@ import org.openmetadata.service.search.SearchRequest; import org.openmetadata.service.search.indexes.ContainerIndex; import org.openmetadata.service.search.indexes.DashboardDataModelIndex; import org.openmetadata.service.search.indexes.DashboardIndex; +import org.openmetadata.service.search.indexes.DataProductIndex; import org.openmetadata.service.search.indexes.DomainIndex; import org.openmetadata.service.search.indexes.GlossaryTermIndex; import org.openmetadata.service.search.indexes.MlModelIndex; @@ -298,6 +299,9 @@ public class OpenSearchClient implements SearchClient { searchSourceBuilder = buildCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize()); break; + case "data_product_search_index": + searchSourceBuilder = buildDataProductSearch(request.getQuery(), request.getFrom(), request.getSize()); + break; default: searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize()); break; @@ -331,7 +335,7 @@ public class OpenSearchClient implements SearchClient { /* For backward-compatibility we continue supporting the deleted argument, this should be removed in future versions */ if (request.getIndex().equalsIgnoreCase("domain_search_index") - || request.getIndex().equalsIgnoreCase("data_products_search_index") + || request.getIndex().equalsIgnoreCase("data_product_search_index") || request.getIndex().equalsIgnoreCase("query_search_index") || request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index") || request.getIndex().equalsIgnoreCase("aggregated_cost_analysis_report_data_index")) { @@ -893,6 +897,32 @@ public class OpenSearchClient implements SearchClient { return builder; } + private static SearchSourceBuilder buildDataProductSearch(String query, int from, int size) { + QueryStringQueryBuilder queryBuilder = + QueryBuilders.queryStringQuery(query) + .fields(DataProductIndex.getFields()) + .defaultOperator(Operator.AND) + .fuzziness(Fuzziness.AUTO); + + HighlightBuilder hb = new HighlightBuilder(); + HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION); + highlightDescription.highlighterType(UNIFIED); + HighlightBuilder.Field highlightName = new HighlightBuilder.Field(FIELD_NAME); + highlightName.highlighterType(UNIFIED); + HighlightBuilder.Field highlightDisplayName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME); + highlightDisplayName.highlighterType(UNIFIED); + hb.field(highlightDescription); + hb.field(highlightName); + hb.field(highlightDisplayName); + + hb.preTags(PRE_TAG); + hb.postTags(POST_TAG); + + SearchSourceBuilder searchSourceBuilder = + new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); + return addAggregation(searchSourceBuilder); + } + private static SearchSourceBuilder searchBuilder(QueryBuilder queryBuilder, HighlightBuilder hb, int from, int size) { SearchSourceBuilder builder = new SearchSourceBuilder().query(queryBuilder).from(from).size(size); if (hb != null) { diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json index 4ff21f96191..e8c5faf2916 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json @@ -155,6 +155,25 @@ } } }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, "tags": { "properties": { "tagFQN": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/glossary_term_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/glossary_term_index_mapping.json index 5dbb11b8544..99a108f485a 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/glossary_term_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/glossary_term_index_mapping.json @@ -415,4 +415,4 @@ } } } - } + }s diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json index 74f4967cacd..79b02a5c0b1 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json @@ -80,6 +80,25 @@ "analyzer": "om_analyzer", "index_options": "freqs" }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, "fields": { "properties": { "name": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json index 46bbed7bd36..97491511fc1 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json @@ -414,6 +414,25 @@ "code": { "type": "keyword" }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, "tags": { "properties": { "tagFQN": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index.json b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index.json index 2f71f68d98b..fdd26ad388d 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index.json @@ -157,6 +157,25 @@ } } }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, "tags": { "properties": { "tagFQN": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json index 738868d2694..fa2e72e5833 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json @@ -82,6 +82,25 @@ "type": "text", "analyzer": "om_analyzer_jp" }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, "fields": { "properties": { "name": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json index 24a37167870..6374a13594a 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json @@ -416,6 +416,25 @@ "code": { "type": "keyword" }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, "tags": { "properties": { "tagFQN": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json index 4a99c927ad4..78bacbbb60d 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json @@ -141,6 +141,25 @@ } } }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, "tags": { "properties": { "tagFQN": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json index 222a44a9678..5cbe43c1c0f 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json @@ -50,6 +50,25 @@ "fqnParts": { "type": "keyword" }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, "fields": { "properties": { "name": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json index 4ee891d777f..e60f96027c8 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json @@ -400,6 +400,25 @@ "code": { "type": "keyword" }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, "tags": { "properties": { "tagFQN": {