yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-03 03:59:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

addded aggreagtions for new indexes and fields and missing things (#13278)

Browse Source
This commit is contained in:
07Himank 2023-09-21 10:41:38 +05:30 committed by GitHub
parent a778500e26
commit 8292ad0078
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 242 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java
View File

@ -11,6 +11,7 @@ public class EntityBuilderConstant {
public static final String COLUMNS_NAME_KEYWORD = "columns.name.keyword";
public static final String OWNER_DISPLAY_NAME_KEYWORD = "owner.displayName.keyword";
public static final String DOMAIN_DISPLAY_NAME_KEYWORD = "domain.displayName.keyword";
public static final String DATA_MODEL_COLUMNS_NAME_KEYWORD = "dataModel.columns.name.keyword";
public static final String NAME_KEYWORD = "name.keyword";
public static final String FIELD_NAME_NGRAM = "name.ngram";

44
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClientImpl.java
View File

@ -8,6 +8,7 @@ import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.Entity.QUERY;
import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE;
@ -615,7 +616,8 @@ public class ElasticSearchClientImpl implements SearchClient {
searchSourceBuilder.aggregation(AggregationBuilders.terms("database.name.keyword").field("database.name.keyword"));
searchSourceBuilder
.aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword"))
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD));
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms("tableType").field("tableType"));
return addAggregation(searchSourceBuilder);
}
@ -653,11 +655,8 @@ public class ElasticSearchClientImpl implements SearchClient {
hb.postTags(POST_TAG);
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
searchSourceBuilder
.aggregation(AggregationBuilders.terms(ES_TAG_FQN_FIELD).field(ES_TAG_FQN_FIELD).size(MAX_AGGREGATE_SIZE))
.aggregation(AggregationBuilders.terms("glossary.name.keyword").field("glossary.name.keyword"))
.aggregation(AggregationBuilders.terms(OWNER_DISPLAY_NAME_KEYWORD).field(OWNER_DISPLAY_NAME_KEYWORD));
return searchSourceBuilder;
searchSourceBuilder.aggregation(AggregationBuilders.terms("glossary.name.keyword").field("glossary.name.keyword"));
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildTagSearchBuilder(String query, int from, int size) {
@ -679,8 +678,11 @@ public class ElasticSearchClientImpl implements SearchClient {
hb.field(highlightTagName);
hb.preTags(PRE_TAG);
hb.postTags(POST_TAG);
return searchBuilder(queryBuilder, hb, from, size)
.aggregation(AggregationBuilders.terms("classification.name.keyword").field("classification.name.keyword"));
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
searchSourceBuilder.aggregation(
AggregationBuilders.terms("classification.name.keyword").field("classification.name.keyword"));
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildContainerSearchBuilder(String query, int from, int size) {
@ -780,8 +782,9 @@ public class ElasticSearchClientImpl implements SearchClient {
hb.preTags(PRE_TAG);
hb.postTags(POST_TAG);
return searchBuilder(queryBuilder, hb, from, size);
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildDashboardDataModelsSearch(String query, int from, int size) {
@ -802,7 +805,13 @@ public class ElasticSearchClientImpl implements SearchClient {
hb.preTags(PRE_TAG);
hb.postTags(POST_TAG);
return searchBuilder(queryBuilder, hb, from, size);
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
searchSourceBuilder
.aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType"))
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword"));
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildDomainsSearch(String query, int from, int size) {
@ -844,7 +853,10 @@ public class ElasticSearchClientImpl implements SearchClient {
hb.preTags(PRE_TAG);
hb.postTags(POST_TAG);
return searchBuilder(queryBuilder, hb, from, size);
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
searchSourceBuilder.aggregation(AggregationBuilders.terms("fields.name.keyword").field("fields.name.keyword"));
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildAggregateSearchBuilder(String query, int from, int size) {
@ -860,11 +872,15 @@ public class ElasticSearchClientImpl implements SearchClient {
AggregationBuilders.terms("service.name.keyword").field("service.name.keyword").size(MAX_AGGREGATE_SIZE))
.aggregation(
AggregationBuilders.terms("entityType.keyword").field("entityType.keyword").size(MAX_AGGREGATE_SIZE))
.aggregation(AggregationBuilders.terms("tier.tagFQN").field("tier.tagFQN"))
.aggregation(AggregationBuilders.terms("tier.tagFQN").field("tier.tagFQN").size(MAX_AGGREGATE_SIZE))
.aggregation(
AggregationBuilders.terms(OWNER_DISPLAY_NAME_KEYWORD)
.field(OWNER_DISPLAY_NAME_KEYWORD)
.size(MAX_AGGREGATE_SIZE))
.aggregation(
AggregationBuilders.terms(DOMAIN_DISPLAY_NAME_KEYWORD)
.field(DOMAIN_DISPLAY_NAME_KEYWORD)
.size(MAX_AGGREGATE_SIZE))
.aggregation(AggregationBuilders.terms(ES_TAG_FQN_FIELD).field(ES_TAG_FQN_FIELD));
return builder;
@ -944,7 +960,7 @@ public class ElasticSearchClientImpl implements SearchClient {
try {
deleteEntityFromElasticSearch(deleteRequest);
if (!CommonUtil.nullOrEmpty(scriptTxt) && !CommonUtil.nullOrEmpty(field)) {
UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest("SearchAlias");
UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(GLOBAL_SEARCH_ALIAS);
updateByQueryRequest.setQuery(new MatchQueryBuilder(field, entity.getFullyQualifiedName()));
updateByQueryRequest.setRefresh(true);
Script script =

12
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java
View File

@ -3,7 +3,11 @@ package org.openmetadata.service.search.indexes;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.*;
import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
import java.util.ArrayList;
import java.util.HashMap;
@ -35,6 +39,12 @@ public class ContainerIndex implements ColumnIndex {
owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName());
container.setOwner(owner);
}
if (container.getDomain() != null) {
EntityReference domain = container.getDomain();
domain.setDisplayName(
CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName());
container.setDomain(domain);
}
Map<String, Object> doc = JsonUtils.getMap(container);
List<SearchSuggest> suggest = new ArrayList<>();
List<SearchSuggest> columnSuggest = new ArrayList<>();

13
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java
View File

@ -3,7 +3,12 @@ package org.openmetadata.service.search.indexes;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.*;
import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
import java.util.ArrayList;
import java.util.HashMap;
@ -34,6 +39,12 @@ public class DashboardDataModelIndex implements ElasticSearchIndex {
owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName());
dashboardDataModel.setOwner(owner);
}
if (dashboardDataModel.getDomain() != null) {
EntityReference domain = dashboardDataModel.getDomain();
domain.setDisplayName(
CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName());
dashboardDataModel.setDomain(domain);
}
Map<String, Object> doc = JsonUtils.getMap(dashboardDataModel);
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
List<SearchSuggest> suggest = new ArrayList<>();

13
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java
View File

@ -4,7 +4,12 @@ import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.*;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
import java.util.ArrayList;
import java.util.HashMap;
@ -34,6 +39,12 @@ public class DashboardIndex implements ElasticSearchIndex {
owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName());
dashboard.setOwner(owner);
}
if (dashboard.getDomain() != null) {
EntityReference domain = dashboard.getDomain();
domain.setDisplayName(
CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName());
dashboard.setDomain(domain);
}
Map<String, Object> doc = JsonUtils.getMap(dashboard);
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
List<SearchSuggest> suggest = new ArrayList<>();

7
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java
View File

@ -1,8 +1,11 @@
package org.openmetadata.service.search.indexes;
import static org.openmetadata.service.Entity.*;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.search.EntityBuilderConstant.*;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
import java.util.ArrayList;
import java.util.HashMap;

11
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java
View File

@ -3,7 +3,10 @@ package org.openmetadata.service.search.indexes;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.*;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
import java.util.ArrayList;
import java.util.HashMap;
@ -32,6 +35,12 @@ public class GlossaryTermIndex implements ElasticSearchIndex {
owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName());
glossaryTerm.setOwner(owner);
}
if (glossaryTerm.getDomain() != null) {
EntityReference domain = glossaryTerm.getDomain();
domain.setDisplayName(
CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName());
glossaryTerm.setDomain(domain);
}
Map<String, Object> doc = JsonUtils.getMap(glossaryTerm);
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
List<SearchSuggest> suggest = new ArrayList<>();

12
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java
View File

@ -3,7 +3,11 @@ package org.openmetadata.service.search.indexes;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.*;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
import java.util.ArrayList;
import java.util.HashMap;
@ -33,6 +37,12 @@ public class MlModelIndex implements ElasticSearchIndex {
owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName());
mlModel.setOwner(owner);
}
if (mlModel.getDomain() != null) {
EntityReference domain = mlModel.getDomain();
domain.setDisplayName(
CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName());
mlModel.setDomain(domain);
}
Map<String, Object> doc = JsonUtils.getMap(mlModel);
List<SearchSuggest> suggest = new ArrayList<>();
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);

12
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java
View File

@ -3,7 +3,11 @@ package org.openmetadata.service.search.indexes;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.*;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
import java.util.ArrayList;
import java.util.HashMap;
@ -34,6 +38,12 @@ public class PipelineIndex implements ElasticSearchIndex {
owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName());
pipeline.setOwner(owner);
}
if (pipeline.getDomain() != null) {
EntityReference domain = pipeline.getDomain();
domain.setDisplayName(
CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName());
pipeline.setDomain(domain);
}
Map<String, Object> doc = JsonUtils.getMap(pipeline);
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
List<SearchSuggest> suggest = new ArrayList<>();

16
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java
View File

@ -1,8 +1,14 @@
package org.openmetadata.service.search.indexes;
import static org.openmetadata.service.Entity.*;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.search.EntityBuilderConstant.*;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
import java.util.ArrayList;
import java.util.HashMap;
@ -32,6 +38,12 @@ public class SearchEntityIndex implements ElasticSearchIndex {
owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName());
searchIndex.setOwner(owner);
}
if (searchIndex.getDomain() != null) {
EntityReference domain = searchIndex.getDomain();
domain.setDisplayName(
CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName());
searchIndex.setDomain(domain);
}
Map<String, Object> doc = JsonUtils.getMap(searchIndex);
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
List<SearchSuggest> suggest = new ArrayList<>();

10
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java
View File

@ -3,7 +3,9 @@ package org.openmetadata.service.search.indexes;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.*;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
import java.util.ArrayList;
import java.util.HashMap;
@ -34,6 +36,12 @@ public class StoredProcedureIndex implements ElasticSearchIndex {
owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName());
storedProcedure.setOwner(owner);
}
if (storedProcedure.getDomain() != null) {
EntityReference domain = storedProcedure.getDomain();
domain.setDisplayName(
CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName());
storedProcedure.setDomain(domain);
}
Map<String, Object> doc = JsonUtils.getMap(storedProcedure);
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
List<SearchSuggest> suggest = new ArrayList<>();

13
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java
View File

@ -3,7 +3,12 @@ package org.openmetadata.service.search.indexes;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.*;
import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
import java.util.ArrayList;
import java.util.HashMap;
@ -42,6 +47,12 @@ public class TableIndex implements ColumnIndex {
owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName());
table.setOwner(owner);
}
if (table.getDomain() != null) {
EntityReference domain = table.getDomain();
domain.setDisplayName(
CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName());
table.setDomain(domain);
}
Map<String, Object> doc = JsonUtils.getMap(table);
List<SearchSuggest> suggest = new ArrayList<>();
List<SearchSuggest> columnSuggest = new ArrayList<>();

8
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java
View File

@ -11,7 +11,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.entity.classification.Tag;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.Entity;
import org.openmetadata.service.search.SearchIndexUtils;
import org.openmetadata.service.search.models.SearchSuggest;
@ -26,6 +28,12 @@ public class TagIndex implements ElasticSearchIndex {
}
public Map<String, Object> buildESDoc() {
if (tag.getDomain() != null) {
EntityReference domain = tag.getDomain();
domain.setDisplayName(
CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName());
tag.setDomain(domain);
}
Map<String, Object> doc = JsonUtils.getMap(tag);
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
List<SearchSuggest> suggest = new ArrayList<>();

14
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java
View File

@ -3,7 +3,13 @@ package org.openmetadata.service.search.indexes;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.*;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
import java.util.ArrayList;
import java.util.HashMap;
@ -39,6 +45,12 @@ public class TopicIndex implements ElasticSearchIndex {
owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName());
topic.setOwner(owner);
}
if (topic.getDomain() != null) {
EntityReference domain = topic.getDomain();
domain.setDisplayName(
CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName());
topic.setDomain(domain);
}
Map<String, Object> doc = JsonUtils.getMap(topic);
List<SearchSuggest> suggest = new ArrayList<>();
List<SearchSuggest> fieldSuggest = new ArrayList<>();

65
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClientImpl.java
View File

@ -8,6 +8,7 @@ import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.Entity.QUERY;
import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE;
@ -69,6 +70,7 @@ import org.openmetadata.service.search.indexes.GlossaryTermIndex;
import org.openmetadata.service.search.indexes.MlModelIndex;
import org.openmetadata.service.search.indexes.PipelineIndex;
import org.openmetadata.service.search.indexes.QueryIndex;
import org.openmetadata.service.search.indexes.SearchEntityIndex;
import org.openmetadata.service.search.indexes.StoredProcedureIndex;
import org.openmetadata.service.search.indexes.TableIndex;
import org.openmetadata.service.search.indexes.TagIndex;
@ -316,6 +318,9 @@ public class OpenSearchClientImpl implements SearchClient {
case "domain_search_index":
searchSourceBuilder = buildDomainsSearch(request.getQuery(), request.getFrom(), request.getSize());
break;
case "search_entity_index":
searchSourceBuilder = buildSearchEntitySearch(request.getQuery(), request.getFrom(), request.getSize());
break;
default:
searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize());
break;
@ -621,8 +626,8 @@ public class OpenSearchClientImpl implements SearchClient {
searchSourceBuilder.aggregation(AggregationBuilders.terms("database.name.keyword").field("database.name.keyword"));
searchSourceBuilder
.aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword"))
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD));
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms("tableType").field("tableType"));
return addAggregation(searchSourceBuilder);
}
@ -660,11 +665,8 @@ public class OpenSearchClientImpl implements SearchClient {
hb.postTags(POST_TAG);
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
searchSourceBuilder
.aggregation(AggregationBuilders.terms(ES_TAG_FQN_FIELD).field(ES_TAG_FQN_FIELD).size(MAX_AGGREGATE_SIZE))
.aggregation(AggregationBuilders.terms("glossary.name.keyword").field("glossary.name.keyword"))
.aggregation(AggregationBuilders.terms(OWNER_DISPLAY_NAME_KEYWORD).field(OWNER_DISPLAY_NAME_KEYWORD));
return searchSourceBuilder;
searchSourceBuilder.aggregation(AggregationBuilders.terms("glossary.name.keyword").field("glossary.name.keyword"));
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildTagSearchBuilder(String query, int from, int size) {
@ -686,8 +688,11 @@ public class OpenSearchClientImpl implements SearchClient {
hb.field(highlightTagName);
hb.preTags(PRE_TAG);
hb.postTags(POST_TAG);
return searchBuilder(queryBuilder, hb, from, size)
.aggregation(AggregationBuilders.terms("classification.name.keyword").field("classification.name.keyword"));
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
searchSourceBuilder.aggregation(
AggregationBuilders.terms("classification.name.keyword").field("classification.name.keyword"));
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildContainerSearchBuilder(String query, int from, int size) {
@ -787,7 +792,9 @@ public class OpenSearchClientImpl implements SearchClient {
hb.preTags(PRE_TAG);
hb.postTags(POST_TAG);
return searchBuilder(queryBuilder, hb, from, size);
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildDashboardDataModelsSearch(String query, int from, int size) {
@ -808,7 +815,13 @@ public class OpenSearchClientImpl implements SearchClient {
hb.preTags(PRE_TAG);
hb.postTags(POST_TAG);
return searchBuilder(queryBuilder, hb, from, size);
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
searchSourceBuilder
.aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType"))
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword"));
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildDomainsSearch(String query, int from, int size) {
@ -832,6 +845,30 @@ public class OpenSearchClientImpl implements SearchClient {
return searchBuilder(queryBuilder, hb, from, size);
}
private static SearchSourceBuilder buildSearchEntitySearch(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.fields(SearchEntityIndex.getFields())
.defaultOperator(Operator.AND)
.fuzziness(Fuzziness.AUTO);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(UNIFIED);
HighlightBuilder.Field highlightName = new HighlightBuilder.Field(FIELD_NAME);
highlightName.highlighterType(UNIFIED);
HighlightBuilder hb = new HighlightBuilder();
hb.field(highlightDescription);
hb.field(highlightName);
hb.preTags(PRE_TAG);
hb.postTags(POST_TAG);
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
searchSourceBuilder.aggregation(AggregationBuilders.terms("fields.name.keyword").field("fields.name.keyword"));
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildAggregateSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query).lenient(true);
SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);
@ -845,11 +882,15 @@ public class OpenSearchClientImpl implements SearchClient {
AggregationBuilders.terms("service.name.keyword").field("service.name.keyword").size(MAX_AGGREGATE_SIZE))
.aggregation(
AggregationBuilders.terms("entityType.keyword").field("entityType.keyword").size(MAX_AGGREGATE_SIZE))
.aggregation(AggregationBuilders.terms("tier.tagFQN").field("tier.tagFQN"))
.aggregation(AggregationBuilders.terms("tier.tagFQN").field("tier.tagFQN").size(MAX_AGGREGATE_SIZE))
.aggregation(
AggregationBuilders.terms(OWNER_DISPLAY_NAME_KEYWORD)
.field(OWNER_DISPLAY_NAME_KEYWORD)
.size(MAX_AGGREGATE_SIZE))
.aggregation(
AggregationBuilders.terms(DOMAIN_DISPLAY_NAME_KEYWORD)
.field(DOMAIN_DISPLAY_NAME_KEYWORD)
.size(MAX_AGGREGATE_SIZE))
.aggregation(AggregationBuilders.terms(ES_TAG_FQN_FIELD).field(ES_TAG_FQN_FIELD));
return builder;
}

8
openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json
View File

@ -272,7 +272,13 @@
}
},
"project": {
"type": "text"
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"lifeCycle": {
"type": "object"

3
openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json
View File

@ -92,6 +92,9 @@
"sourceUrl": {
"type": "keyword"
},
"tableType": {
"type": "keyword"
},
"columns": {
"properties": {
"name": {

8
openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index.json
View File

@ -280,7 +280,13 @@
}
},
"project": {
"type": "text"
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"lifeCycle": {
"type": "object"

3
openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json
View File

@ -111,6 +111,9 @@
"sourceUrl": {
"type": "keyword"
},
"tableType": {
"type": "keyword"
},
"columns": {
"properties": {
"name": {

8
openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json
View File

@ -259,7 +259,13 @@
}
},
"project": {
"type": "text"
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"lifeCycle": {
"type": "object"

3
openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json
View File

@ -76,6 +76,9 @@
"sourceUrl": {
"type": "keyword"
},
"tableType": {
"type": "keyword"
},
"columns": {
"properties": {
"name": {