From 8292ad00788a86559bc0a8d43c19b72d1b73a9bc Mon Sep 17 00:00:00 2001 From: 07Himank <112613760+07Himank@users.noreply.github.com> Date: Thu, 21 Sep 2023 10:41:38 +0530 Subject: [PATCH] addded aggreagtions for new indexes and fields and missing things (#13278) --- .../service/search/EntityBuilderConstant.java | 1 + .../ElasticSearchClientImpl.java | 44 +++++++++---- .../search/indexes/ContainerIndex.java | 12 +++- .../indexes/DashboardDataModelIndex.java | 13 +++- .../search/indexes/DashboardIndex.java | 13 +++- .../service/search/indexes/DomainIndex.java | 7 +- .../search/indexes/GlossaryTermIndex.java | 11 +++- .../service/search/indexes/MlModelIndex.java | 12 +++- .../service/search/indexes/PipelineIndex.java | 12 +++- .../search/indexes/SearchEntityIndex.java | 16 ++++- .../search/indexes/StoredProcedureIndex.java | 10 ++- .../service/search/indexes/TableIndex.java | 13 +++- .../service/search/indexes/TagIndex.java | 8 +++ .../service/search/indexes/TopicIndex.java | 14 +++- .../opensearch/OpenSearchClientImpl.java | 65 +++++++++++++++---- .../dashboard_data_model_index_mapping.json | 8 ++- .../elasticsearch/en/table_index_mapping.json | 3 + .../jp/dashboard_data_model_index.json | 8 ++- .../elasticsearch/jp/table_index_mapping.json | 3 + .../dashboard_data_model_index_mapping.json | 8 ++- .../elasticsearch/zh/table_index_mapping.json | 3 + 21 files changed, 242 insertions(+), 42 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java index b7e3a2b826f..882b18824e6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java @@ -11,6 +11,7 @@ public class EntityBuilderConstant { public static final String COLUMNS_NAME_KEYWORD = "columns.name.keyword"; public static final String OWNER_DISPLAY_NAME_KEYWORD = "owner.displayName.keyword"; + public static final String DOMAIN_DISPLAY_NAME_KEYWORD = "domain.displayName.keyword"; public static final String DATA_MODEL_COLUMNS_NAME_KEYWORD = "dataModel.columns.name.keyword"; public static final String NAME_KEYWORD = "name.keyword"; public static final String FIELD_NAME_NGRAM = "name.ngram"; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClientImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClientImpl.java index 5318b00bf1d..a42f07107b0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClientImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClientImpl.java @@ -8,6 +8,7 @@ import static org.openmetadata.service.Entity.FIELD_NAME; import static org.openmetadata.service.Entity.QUERY; import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD; import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD; import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; @@ -615,7 +616,8 @@ public class ElasticSearchClientImpl implements SearchClient { searchSourceBuilder.aggregation(AggregationBuilders.terms("database.name.keyword").field("database.name.keyword")); searchSourceBuilder .aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword")) - .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)); + .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) + .aggregation(AggregationBuilders.terms("tableType").field("tableType")); return addAggregation(searchSourceBuilder); } @@ -653,11 +655,8 @@ public class ElasticSearchClientImpl implements SearchClient { hb.postTags(POST_TAG); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); - searchSourceBuilder - .aggregation(AggregationBuilders.terms(ES_TAG_FQN_FIELD).field(ES_TAG_FQN_FIELD).size(MAX_AGGREGATE_SIZE)) - .aggregation(AggregationBuilders.terms("glossary.name.keyword").field("glossary.name.keyword")) - .aggregation(AggregationBuilders.terms(OWNER_DISPLAY_NAME_KEYWORD).field(OWNER_DISPLAY_NAME_KEYWORD)); - return searchSourceBuilder; + searchSourceBuilder.aggregation(AggregationBuilders.terms("glossary.name.keyword").field("glossary.name.keyword")); + return addAggregation(searchSourceBuilder); } private static SearchSourceBuilder buildTagSearchBuilder(String query, int from, int size) { @@ -679,8 +678,11 @@ public class ElasticSearchClientImpl implements SearchClient { hb.field(highlightTagName); hb.preTags(PRE_TAG); hb.postTags(POST_TAG); - return searchBuilder(queryBuilder, hb, from, size) - .aggregation(AggregationBuilders.terms("classification.name.keyword").field("classification.name.keyword")); + SearchSourceBuilder searchSourceBuilder = + new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); + searchSourceBuilder.aggregation( + AggregationBuilders.terms("classification.name.keyword").field("classification.name.keyword")); + return addAggregation(searchSourceBuilder); } private static SearchSourceBuilder buildContainerSearchBuilder(String query, int from, int size) { @@ -780,8 +782,9 @@ public class ElasticSearchClientImpl implements SearchClient { hb.preTags(PRE_TAG); hb.postTags(POST_TAG); - - return searchBuilder(queryBuilder, hb, from, size); + SearchSourceBuilder searchSourceBuilder = + new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); + return addAggregation(searchSourceBuilder); } private static SearchSourceBuilder buildDashboardDataModelsSearch(String query, int from, int size) { @@ -802,7 +805,13 @@ public class ElasticSearchClientImpl implements SearchClient { hb.preTags(PRE_TAG); hb.postTags(POST_TAG); - return searchBuilder(queryBuilder, hb, from, size); + SearchSourceBuilder searchSourceBuilder = + new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); + searchSourceBuilder + .aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType")) + .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) + .aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword")); + return addAggregation(searchSourceBuilder); } private static SearchSourceBuilder buildDomainsSearch(String query, int from, int size) { @@ -844,7 +853,10 @@ public class ElasticSearchClientImpl implements SearchClient { hb.preTags(PRE_TAG); hb.postTags(POST_TAG); - return searchBuilder(queryBuilder, hb, from, size); + SearchSourceBuilder searchSourceBuilder = + new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); + searchSourceBuilder.aggregation(AggregationBuilders.terms("fields.name.keyword").field("fields.name.keyword")); + return addAggregation(searchSourceBuilder); } private static SearchSourceBuilder buildAggregateSearchBuilder(String query, int from, int size) { @@ -860,11 +872,15 @@ public class ElasticSearchClientImpl implements SearchClient { AggregationBuilders.terms("service.name.keyword").field("service.name.keyword").size(MAX_AGGREGATE_SIZE)) .aggregation( AggregationBuilders.terms("entityType.keyword").field("entityType.keyword").size(MAX_AGGREGATE_SIZE)) - .aggregation(AggregationBuilders.terms("tier.tagFQN").field("tier.tagFQN")) + .aggregation(AggregationBuilders.terms("tier.tagFQN").field("tier.tagFQN").size(MAX_AGGREGATE_SIZE)) .aggregation( AggregationBuilders.terms(OWNER_DISPLAY_NAME_KEYWORD) .field(OWNER_DISPLAY_NAME_KEYWORD) .size(MAX_AGGREGATE_SIZE)) + .aggregation( + AggregationBuilders.terms(DOMAIN_DISPLAY_NAME_KEYWORD) + .field(DOMAIN_DISPLAY_NAME_KEYWORD) + .size(MAX_AGGREGATE_SIZE)) .aggregation(AggregationBuilders.terms(ES_TAG_FQN_FIELD).field(ES_TAG_FQN_FIELD)); return builder; @@ -944,7 +960,7 @@ public class ElasticSearchClientImpl implements SearchClient { try { deleteEntityFromElasticSearch(deleteRequest); if (!CommonUtil.nullOrEmpty(scriptTxt) && !CommonUtil.nullOrEmpty(field)) { - UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest("SearchAlias"); + UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(GLOBAL_SEARCH_ALIAS); updateByQueryRequest.setQuery(new MatchQueryBuilder(field, entity.getFullyQualifiedName())); updateByQueryRequest.setRefresh(true); Script script = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java index 099c8ce7971..7d3066832be 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java @@ -3,7 +3,11 @@ package org.openmetadata.service.search.indexes; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.search.EntityBuilderConstant.*; +import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; import java.util.HashMap; @@ -35,6 +39,12 @@ public class ContainerIndex implements ColumnIndex { owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName()); container.setOwner(owner); } + if (container.getDomain() != null) { + EntityReference domain = container.getDomain(); + domain.setDisplayName( + CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName()); + container.setDomain(domain); + } Map doc = JsonUtils.getMap(container); List suggest = new ArrayList<>(); List columnSuggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java index 1fab139ea05..98c6581a002 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java @@ -3,7 +3,12 @@ package org.openmetadata.service.search.indexes; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.search.EntityBuilderConstant.*; +import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; import java.util.HashMap; @@ -34,6 +39,12 @@ public class DashboardDataModelIndex implements ElasticSearchIndex { owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName()); dashboardDataModel.setOwner(owner); } + if (dashboardDataModel.getDomain() != null) { + EntityReference domain = dashboardDataModel.getDomain(); + domain.setDisplayName( + CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName()); + dashboardDataModel.setDomain(domain); + } Map doc = JsonUtils.getMap(dashboardDataModel); SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); List suggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java index 318da0b5334..af00df996e7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java @@ -4,7 +4,12 @@ import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.search.EntityBuilderConstant.*; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; import java.util.HashMap; @@ -34,6 +39,12 @@ public class DashboardIndex implements ElasticSearchIndex { owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName()); dashboard.setOwner(owner); } + if (dashboard.getDomain() != null) { + EntityReference domain = dashboard.getDomain(); + domain.setDisplayName( + CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName()); + dashboard.setDomain(domain); + } Map doc = JsonUtils.getMap(dashboard); SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); List suggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java index d809e36af11..f30d97bedd6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java @@ -1,8 +1,11 @@ package org.openmetadata.service.search.indexes; -import static org.openmetadata.service.Entity.*; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; -import static org.openmetadata.service.search.EntityBuilderConstant.*; +import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; +import static org.openmetadata.service.Entity.FIELD_NAME; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; import java.util.HashMap; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java index 6387079af7b..7547b60d347 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java @@ -3,7 +3,10 @@ package org.openmetadata.service.search.indexes; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.search.EntityBuilderConstant.*; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; import java.util.HashMap; @@ -32,6 +35,12 @@ public class GlossaryTermIndex implements ElasticSearchIndex { owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName()); glossaryTerm.setOwner(owner); } + if (glossaryTerm.getDomain() != null) { + EntityReference domain = glossaryTerm.getDomain(); + domain.setDisplayName( + CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName()); + glossaryTerm.setDomain(domain); + } Map doc = JsonUtils.getMap(glossaryTerm); SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); List suggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java index 5eb837bc0c3..94eb8045c08 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java @@ -3,7 +3,11 @@ package org.openmetadata.service.search.indexes; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.search.EntityBuilderConstant.*; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; import java.util.HashMap; @@ -33,6 +37,12 @@ public class MlModelIndex implements ElasticSearchIndex { owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName()); mlModel.setOwner(owner); } + if (mlModel.getDomain() != null) { + EntityReference domain = mlModel.getDomain(); + domain.setDisplayName( + CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName()); + mlModel.setDomain(domain); + } Map doc = JsonUtils.getMap(mlModel); List suggest = new ArrayList<>(); SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java index 596822f5912..73ad40b41f1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java @@ -3,7 +3,11 @@ package org.openmetadata.service.search.indexes; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.search.EntityBuilderConstant.*; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; import java.util.HashMap; @@ -34,6 +38,12 @@ public class PipelineIndex implements ElasticSearchIndex { owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName()); pipeline.setOwner(owner); } + if (pipeline.getDomain() != null) { + EntityReference domain = pipeline.getDomain(); + domain.setDisplayName( + CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName()); + pipeline.setDomain(domain); + } Map doc = JsonUtils.getMap(pipeline); SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); List suggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java index bb363d5ce0c..0e3aa3a5c78 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java @@ -1,8 +1,14 @@ package org.openmetadata.service.search.indexes; -import static org.openmetadata.service.Entity.*; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; -import static org.openmetadata.service.search.EntityBuilderConstant.*; +import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; +import static org.openmetadata.service.Entity.FIELD_NAME; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; import java.util.HashMap; @@ -32,6 +38,12 @@ public class SearchEntityIndex implements ElasticSearchIndex { owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName()); searchIndex.setOwner(owner); } + if (searchIndex.getDomain() != null) { + EntityReference domain = searchIndex.getDomain(); + domain.setDisplayName( + CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName()); + searchIndex.setDomain(domain); + } Map doc = JsonUtils.getMap(searchIndex); SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); List suggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java index 3c0e4eea75f..e1d4bc991ba 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java @@ -3,7 +3,9 @@ package org.openmetadata.service.search.indexes; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.search.EntityBuilderConstant.*; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; import java.util.HashMap; @@ -34,6 +36,12 @@ public class StoredProcedureIndex implements ElasticSearchIndex { owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName()); storedProcedure.setOwner(owner); } + if (storedProcedure.getDomain() != null) { + EntityReference domain = storedProcedure.getDomain(); + domain.setDisplayName( + CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName()); + storedProcedure.setDomain(domain); + } Map doc = JsonUtils.getMap(storedProcedure); SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); List suggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java index 0c8f3641eed..edfb56cd9ec 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java @@ -3,7 +3,12 @@ package org.openmetadata.service.search.indexes; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.search.EntityBuilderConstant.*; +import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; import java.util.HashMap; @@ -42,6 +47,12 @@ public class TableIndex implements ColumnIndex { owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName()); table.setOwner(owner); } + if (table.getDomain() != null) { + EntityReference domain = table.getDomain(); + domain.setDisplayName( + CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName()); + table.setDomain(domain); + } Map doc = JsonUtils.getMap(table); List suggest = new ArrayList<>(); List columnSuggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java index d028ad4edd7..942b5881c48 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java @@ -11,7 +11,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.entity.classification.Tag; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; @@ -26,6 +28,12 @@ public class TagIndex implements ElasticSearchIndex { } public Map buildESDoc() { + if (tag.getDomain() != null) { + EntityReference domain = tag.getDomain(); + domain.setDisplayName( + CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName()); + tag.setDomain(domain); + } Map doc = JsonUtils.getMap(tag); SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); List suggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java index e12edbd837c..d55e1539a34 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java @@ -3,7 +3,13 @@ package org.openmetadata.service.search.indexes; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.search.EntityBuilderConstant.*; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; import java.util.HashMap; @@ -39,6 +45,12 @@ public class TopicIndex implements ElasticSearchIndex { owner.setDisplayName(CommonUtil.nullOrEmpty(owner.getDisplayName()) ? owner.getName() : owner.getDisplayName()); topic.setOwner(owner); } + if (topic.getDomain() != null) { + EntityReference domain = topic.getDomain(); + domain.setDisplayName( + CommonUtil.nullOrEmpty(domain.getDisplayName()) ? domain.getName() : domain.getDisplayName()); + topic.setDomain(domain); + } Map doc = JsonUtils.getMap(topic); List suggest = new ArrayList<>(); List fieldSuggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClientImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClientImpl.java index 4053fe6f201..aec7244b753 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClientImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClientImpl.java @@ -8,6 +8,7 @@ import static org.openmetadata.service.Entity.FIELD_NAME; import static org.openmetadata.service.Entity.QUERY; import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD; import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD; import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; @@ -69,6 +70,7 @@ import org.openmetadata.service.search.indexes.GlossaryTermIndex; import org.openmetadata.service.search.indexes.MlModelIndex; import org.openmetadata.service.search.indexes.PipelineIndex; import org.openmetadata.service.search.indexes.QueryIndex; +import org.openmetadata.service.search.indexes.SearchEntityIndex; import org.openmetadata.service.search.indexes.StoredProcedureIndex; import org.openmetadata.service.search.indexes.TableIndex; import org.openmetadata.service.search.indexes.TagIndex; @@ -316,6 +318,9 @@ public class OpenSearchClientImpl implements SearchClient { case "domain_search_index": searchSourceBuilder = buildDomainsSearch(request.getQuery(), request.getFrom(), request.getSize()); break; + case "search_entity_index": + searchSourceBuilder = buildSearchEntitySearch(request.getQuery(), request.getFrom(), request.getSize()); + break; default: searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize()); break; @@ -621,8 +626,8 @@ public class OpenSearchClientImpl implements SearchClient { searchSourceBuilder.aggregation(AggregationBuilders.terms("database.name.keyword").field("database.name.keyword")); searchSourceBuilder .aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword")) - .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)); - + .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) + .aggregation(AggregationBuilders.terms("tableType").field("tableType")); return addAggregation(searchSourceBuilder); } @@ -660,11 +665,8 @@ public class OpenSearchClientImpl implements SearchClient { hb.postTags(POST_TAG); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); - searchSourceBuilder - .aggregation(AggregationBuilders.terms(ES_TAG_FQN_FIELD).field(ES_TAG_FQN_FIELD).size(MAX_AGGREGATE_SIZE)) - .aggregation(AggregationBuilders.terms("glossary.name.keyword").field("glossary.name.keyword")) - .aggregation(AggregationBuilders.terms(OWNER_DISPLAY_NAME_KEYWORD).field(OWNER_DISPLAY_NAME_KEYWORD)); - return searchSourceBuilder; + searchSourceBuilder.aggregation(AggregationBuilders.terms("glossary.name.keyword").field("glossary.name.keyword")); + return addAggregation(searchSourceBuilder); } private static SearchSourceBuilder buildTagSearchBuilder(String query, int from, int size) { @@ -686,8 +688,11 @@ public class OpenSearchClientImpl implements SearchClient { hb.field(highlightTagName); hb.preTags(PRE_TAG); hb.postTags(POST_TAG); - return searchBuilder(queryBuilder, hb, from, size) - .aggregation(AggregationBuilders.terms("classification.name.keyword").field("classification.name.keyword")); + SearchSourceBuilder searchSourceBuilder = + new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); + searchSourceBuilder.aggregation( + AggregationBuilders.terms("classification.name.keyword").field("classification.name.keyword")); + return addAggregation(searchSourceBuilder); } private static SearchSourceBuilder buildContainerSearchBuilder(String query, int from, int size) { @@ -787,7 +792,9 @@ public class OpenSearchClientImpl implements SearchClient { hb.preTags(PRE_TAG); hb.postTags(POST_TAG); - return searchBuilder(queryBuilder, hb, from, size); + SearchSourceBuilder searchSourceBuilder = + new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); + return addAggregation(searchSourceBuilder); } private static SearchSourceBuilder buildDashboardDataModelsSearch(String query, int from, int size) { @@ -808,7 +815,13 @@ public class OpenSearchClientImpl implements SearchClient { hb.preTags(PRE_TAG); hb.postTags(POST_TAG); - return searchBuilder(queryBuilder, hb, from, size); + SearchSourceBuilder searchSourceBuilder = + new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); + searchSourceBuilder + .aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType")) + .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) + .aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword")); + return addAggregation(searchSourceBuilder); } private static SearchSourceBuilder buildDomainsSearch(String query, int from, int size) { @@ -832,6 +845,30 @@ public class OpenSearchClientImpl implements SearchClient { return searchBuilder(queryBuilder, hb, from, size); } + private static SearchSourceBuilder buildSearchEntitySearch(String query, int from, int size) { + QueryStringQueryBuilder queryBuilder = + QueryBuilders.queryStringQuery(query) + .fields(SearchEntityIndex.getFields()) + .defaultOperator(Operator.AND) + .fuzziness(Fuzziness.AUTO); + + HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION); + highlightDescription.highlighterType(UNIFIED); + HighlightBuilder.Field highlightName = new HighlightBuilder.Field(FIELD_NAME); + highlightName.highlighterType(UNIFIED); + HighlightBuilder hb = new HighlightBuilder(); + hb.field(highlightDescription); + hb.field(highlightName); + + hb.preTags(PRE_TAG); + hb.postTags(POST_TAG); + + SearchSourceBuilder searchSourceBuilder = + new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); + searchSourceBuilder.aggregation(AggregationBuilders.terms("fields.name.keyword").field("fields.name.keyword")); + return addAggregation(searchSourceBuilder); + } + private static SearchSourceBuilder buildAggregateSearchBuilder(String query, int from, int size) { QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query).lenient(true); SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size); @@ -845,11 +882,15 @@ public class OpenSearchClientImpl implements SearchClient { AggregationBuilders.terms("service.name.keyword").field("service.name.keyword").size(MAX_AGGREGATE_SIZE)) .aggregation( AggregationBuilders.terms("entityType.keyword").field("entityType.keyword").size(MAX_AGGREGATE_SIZE)) - .aggregation(AggregationBuilders.terms("tier.tagFQN").field("tier.tagFQN")) + .aggregation(AggregationBuilders.terms("tier.tagFQN").field("tier.tagFQN").size(MAX_AGGREGATE_SIZE)) .aggregation( AggregationBuilders.terms(OWNER_DISPLAY_NAME_KEYWORD) .field(OWNER_DISPLAY_NAME_KEYWORD) .size(MAX_AGGREGATE_SIZE)) + .aggregation( + AggregationBuilders.terms(DOMAIN_DISPLAY_NAME_KEYWORD) + .field(DOMAIN_DISPLAY_NAME_KEYWORD) + .size(MAX_AGGREGATE_SIZE)) .aggregation(AggregationBuilders.terms(ES_TAG_FQN_FIELD).field(ES_TAG_FQN_FIELD)); return builder; } diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json index 1456d542222..472a563c638 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json @@ -272,7 +272,13 @@ } }, "project": { - "type": "text" + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } }, "lifeCycle": { "type": "object" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json index e9c7b6f7160..4c52dbb0041 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json @@ -92,6 +92,9 @@ "sourceUrl": { "type": "keyword" }, + "tableType": { + "type": "keyword" + }, "columns": { "properties": { "name": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index.json b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index.json index 357d76f47ce..26e4eca9681 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index.json @@ -280,7 +280,13 @@ } }, "project": { - "type": "text" + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } }, "lifeCycle": { "type": "object" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json index d1a466bb3e9..7b92c6d43a3 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json @@ -111,6 +111,9 @@ "sourceUrl": { "type": "keyword" }, + "tableType": { + "type": "keyword" + }, "columns": { "properties": { "name": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json index 21289e477cc..7a53ff861b0 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json @@ -259,7 +259,13 @@ } }, "project": { - "type": "text" + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } }, "lifeCycle": { "type": "object" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json index c7d4520734b..76b49f212da 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json @@ -76,6 +76,9 @@ "sourceUrl": { "type": "keyword" }, + "tableType": { + "type": "keyword" + }, "columns": { "properties": { "name": {