yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-25 08:50:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added missing search things (#13583)

Browse Source 
* missing search things

* revert config local setting

* revert gs term mapping

* fix py test
This commit is contained in:
07Himank 2023-10-16 19:51:11 +05:30 committed by GitHub
parent cdb94e476b
commit cf94097e3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 267 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -136,6 +136,7 @@ import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.Elas
import org.openmetadata.service.search.indexes.ContainerIndex;
import org.openmetadata.service.search.indexes.DashboardDataModelIndex;
import org.openmetadata.service.search.indexes.DashboardIndex;
import org.openmetadata.service.search.indexes.DataProductIndex;
import org.openmetadata.service.search.indexes.DomainIndex;
import org.openmetadata.service.search.indexes.GlossaryTermIndex;
import org.openmetadata.service.search.indexes.MlModelIndex;
@ -306,6 +307,9 @@ public class ElasticSearchClient implements SearchClient {
searchSourceBuilder =
buildCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize());
break;
case "data_product_search_index":
searchSourceBuilder = buildDataProductSearch(request.getQuery(), request.getFrom(), request.getSize());
break;
default:
searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize());
break;
@ -339,7 +343,7 @@ public class ElasticSearchClient implements SearchClient {
/* For backward-compatibility we continue supporting the deleted argument, this should be removed in future versions */
if (request.getIndex().equalsIgnoreCase("domain_search_index")
|| request.getIndex().equalsIgnoreCase("data_products_search_index")
|| request.getIndex().equalsIgnoreCase("data_product_search_index")
|| request.getIndex().equalsIgnoreCase("query_search_index")
|| request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index")
|| request.getIndex().equalsIgnoreCase("aggregated_cost_analysis_report_data_index")) {
@ -855,6 +859,33 @@ public class ElasticSearchClient implements SearchClient {
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildDataProductSearch(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.fields(DataProductIndex.getFields())
.defaultOperator(Operator.AND)
.fuzziness(Fuzziness.AUTO);
HighlightBuilder hb = new HighlightBuilder();
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(UNIFIED);
HighlightBuilder.Field highlightName = new HighlightBuilder.Field(FIELD_NAME);
highlightName.highlighterType(UNIFIED);
HighlightBuilder.Field highlightDisplayName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME);
highlightDisplayName.highlighterType(UNIFIED);
hb.field(highlightDescription);
hb.field(highlightName);
hb.field(highlightDisplayName);
hb.preTags(PRE_TAG);
hb.postTags(POST_TAG);
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildAggregateSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query).lenient(true);
SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);

1
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java
View File

@ -71,6 +71,7 @@ public class DashboardDataModelIndex implements ColumnIndex {
getFQNParts(
dashboardDataModel.getFullyQualifiedName(),
suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
doc.put("tier", parseTags.getTierTag());
if (dashboardDataModel.getOwner() != null) {
doc.put("owner", getOwnerWithDisplayName(dashboardDataModel.getOwner()));
}

23
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java
View File

@ -1,6 +1,16 @@
package org.openmetadata.service.search.indexes;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@ -40,4 +50,17 @@ public class DataProductIndex implements SearchIndex {
}
return doc;
}
public static Map<String, Float> getFields() {
Map<String, Float> fields = new HashMap<>();
fields.put(FIELD_DISPLAY_NAME, 15.0f);
fields.put(FIELD_DISPLAY_NAME_NGRAM, 1.0f);
fields.put(FIELD_NAME, 15.0f);
fields.put(FIELD_NAME_NGRAM, 1.0f);
fields.put(DISPLAY_NAME_KEYWORD, 25.0f);
fields.put(NAME_KEYWORD, 25.0f);
fields.put(FULLY_QUALIFIED_NAME_PARTS, 10.0f);
fields.put(FIELD_DESCRIPTION, 1.0f);
return fields;
}
}

4
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java
View File

@ -15,6 +15,7 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.openmetadata.service.Entity;
import org.openmetadata.service.search.ParseTags;
import org.openmetadata.service.search.SearchIndexUtils;
import org.openmetadata.service.search.models.SearchSuggest;
import org.openmetadata.service.util.JsonUtils;
@ -42,6 +43,9 @@ public class SearchEntityIndex implements SearchIndex {
getFQNParts(
searchIndex.getFullyQualifiedName(),
suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.SEARCH_INDEX, searchIndex));
doc.put("tags", parseTags.getTags());
doc.put("tier", parseTags.getTierTag());
if (searchIndex.getOwner() != null) {
doc.put("owner", getOwnerWithDisplayName(searchIndex.getOwner()));
}

4
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java
View File

@ -14,6 +14,7 @@ import java.util.Map;
import java.util.stream.Collectors;
import org.openmetadata.schema.entity.data.StoredProcedure;
import org.openmetadata.service.Entity;
import org.openmetadata.service.search.ParseTags;
import org.openmetadata.service.search.SearchIndexUtils;
import org.openmetadata.service.search.models.SearchSuggest;
import org.openmetadata.service.util.JsonUtils;
@ -41,6 +42,9 @@ public class StoredProcedureIndex implements SearchIndex {
suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
doc.put("suggest", suggest);
doc.put("entityType", Entity.STORED_PROCEDURE);
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.STORED_PROCEDURE, storedProcedure));
doc.put("tags", parseTags.getTags());
doc.put("tier", parseTags.getTierTag());
if (storedProcedure.getOwner() != null) {
doc.put("owner", getOwnerWithDisplayName(storedProcedure.getOwner()));
}

32
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -51,6 +51,7 @@ import org.openmetadata.service.search.SearchRequest;
import org.openmetadata.service.search.indexes.ContainerIndex;
import org.openmetadata.service.search.indexes.DashboardDataModelIndex;
import org.openmetadata.service.search.indexes.DashboardIndex;
import org.openmetadata.service.search.indexes.DataProductIndex;
import org.openmetadata.service.search.indexes.DomainIndex;
import org.openmetadata.service.search.indexes.GlossaryTermIndex;
import org.openmetadata.service.search.indexes.MlModelIndex;
@ -298,6 +299,9 @@ public class OpenSearchClient implements SearchClient {
searchSourceBuilder =
buildCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize());
break;
case "data_product_search_index":
searchSourceBuilder = buildDataProductSearch(request.getQuery(), request.getFrom(), request.getSize());
break;
default:
searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize());
break;
@ -331,7 +335,7 @@ public class OpenSearchClient implements SearchClient {
/* For backward-compatibility we continue supporting the deleted argument, this should be removed in future versions */
if (request.getIndex().equalsIgnoreCase("domain_search_index")
|| request.getIndex().equalsIgnoreCase("data_products_search_index")
|| request.getIndex().equalsIgnoreCase("data_product_search_index")
|| request.getIndex().equalsIgnoreCase("query_search_index")
|| request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index")
|| request.getIndex().equalsIgnoreCase("aggregated_cost_analysis_report_data_index")) {
@ -893,6 +897,32 @@ public class OpenSearchClient implements SearchClient {
return builder;
}
private static SearchSourceBuilder buildDataProductSearch(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.fields(DataProductIndex.getFields())
.defaultOperator(Operator.AND)
.fuzziness(Fuzziness.AUTO);
HighlightBuilder hb = new HighlightBuilder();
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(UNIFIED);
HighlightBuilder.Field highlightName = new HighlightBuilder.Field(FIELD_NAME);
highlightName.highlighterType(UNIFIED);
HighlightBuilder.Field highlightDisplayName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME);
highlightDisplayName.highlighterType(UNIFIED);
hb.field(highlightDescription);
hb.field(highlightName);
hb.field(highlightDisplayName);
hb.preTags(PRE_TAG);
hb.postTags(POST_TAG);
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder searchBuilder(QueryBuilder queryBuilder, HighlightBuilder hb, int from, int size) {
SearchSourceBuilder builder = new SearchSourceBuilder().query(queryBuilder).from(from).size(size);
if (hb != null) {

19
openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json
View File

@ -155,6 +155,25 @@
}
}
},
"tier": {
"properties": {
"tagFQN": {
"type": "keyword"
},
"labelType": {
"type": "keyword"
},
"description": {
"type": "text"
},
"source": {
"type": "keyword"
},
"state": {
"type": "keyword"
}
}
},
"tags": {
"properties": {
"tagFQN": {

2
openmetadata-service/src/main/resources/elasticsearch/en/glossary_term_index_mapping.json
View File

@ -415,4 +415,4 @@
}
}
}
}
}s

19
openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json
View File

@ -80,6 +80,25 @@
"analyzer": "om_analyzer",
"index_options": "freqs"
},
"tier": {
"properties": {
"tagFQN": {
"type": "keyword"
},
"labelType": {
"type": "keyword"
},
"description": {
"type": "text"
},
"source": {
"type": "keyword"
},
"state": {
"type": "keyword"
}
}
},
"fields": {
"properties": {
"name": {

19
openmetadata-service/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json
View File

@ -414,6 +414,25 @@
"code": {
"type": "keyword"
},
"tier": {
"properties": {
"tagFQN": {
"type": "keyword"
},
"labelType": {
"type": "keyword"
},
"description": {
"type": "text"
},
"source": {
"type": "keyword"
},
"state": {
"type": "keyword"
}
}
},
"tags": {
"properties": {
"tagFQN": {

19
openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index.json
View File

@ -157,6 +157,25 @@
}
}
},
"tier": {
"properties": {
"tagFQN": {
"type": "keyword"
},
"labelType": {
"type": "keyword"
},
"description": {
"type": "text"
},
"source": {
"type": "keyword"
},
"state": {
"type": "keyword"
}
}
},
"tags": {
"properties": {
"tagFQN": {

19
openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json
View File

@ -82,6 +82,25 @@
"type": "text",
"analyzer": "om_analyzer_jp"
},
"tier": {
"properties": {
"tagFQN": {
"type": "keyword"
},
"labelType": {
"type": "keyword"
},
"description": {
"type": "text"
},
"source": {
"type": "keyword"
},
"state": {
"type": "keyword"
}
}
},
"fields": {
"properties": {
"name": {

19
openmetadata-service/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json
View File

@ -416,6 +416,25 @@
"code": {
"type": "keyword"
},
"tier": {
"properties": {
"tagFQN": {
"type": "keyword"
},
"labelType": {
"type": "keyword"
},
"description": {
"type": "text"
},
"source": {
"type": "keyword"
},
"state": {
"type": "keyword"
}
}
},
"tags": {
"properties": {
"tagFQN": {

19
openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json
View File

@ -141,6 +141,25 @@
}
}
},
"tier": {
"properties": {
"tagFQN": {
"type": "keyword"
},
"labelType": {
"type": "keyword"
},
"description": {
"type": "text"
},
"source": {
"type": "keyword"
},
"state": {
"type": "keyword"
}
}
},
"tags": {
"properties": {
"tagFQN": {

19
openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json
View File

@ -50,6 +50,25 @@
"fqnParts": {
"type": "keyword"
},
"tier": {
"properties": {
"tagFQN": {
"type": "keyword"
},
"labelType": {
"type": "keyword"
},
"description": {
"type": "text"
},
"source": {
"type": "keyword"
},
"state": {
"type": "keyword"
}
}
},
"fields": {
"properties": {
"name": {

19
openmetadata-service/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json
View File

@ -400,6 +400,25 @@
"code": {
"type": "keyword"
},
"tier": {
"properties": {
"tagFQN": {
"type": "keyword"
},
"labelType": {
"type": "keyword"
},
"description": {
"type": "text"
},
"source": {
"type": "keyword"
},
"state": {
"type": "keyword"
}
}
},
"tags": {
"properties": {
"tagFQN": {