mirror of
				https://github.com/open-metadata/OpenMetadata.git
				synced 2025-10-26 00:04:52 +00:00 
			
		
		
		
	Nested columns search fix (#13539)
* nested columns work * working on nested columns and its nested tags * worked on nested columns and tags * removed unneccesary variable * use only name in arr field * changed the varable names * added nested columns logic for container and dashboard datamodel too * created constants
This commit is contained in:
		
							parent
							
								
									af9cdc4278
								
							
						
					
					
						commit
						2ef49d52ca
					
				| @ -10,6 +10,8 @@ public class EntityBuilderConstant { | |||||||
|   public static final String ES_TAG_FQN_FIELD = "tags.tagFQN"; |   public static final String ES_TAG_FQN_FIELD = "tags.tagFQN"; | ||||||
| 
 | 
 | ||||||
|   public static final String COLUMNS_NAME_KEYWORD = "columns.name.keyword"; |   public static final String COLUMNS_NAME_KEYWORD = "columns.name.keyword"; | ||||||
|  |   public static final String FIELD_COLUMN_NAMES = "columnNames"; | ||||||
|  |   public static final String SCHEMA_FIELD_NAMES = "fieldNames"; | ||||||
|   public static final String OWNER_DISPLAY_NAME_KEYWORD = "owner.displayName.keyword"; |   public static final String OWNER_DISPLAY_NAME_KEYWORD = "owner.displayName.keyword"; | ||||||
|   public static final String DOMAIN_DISPLAY_NAME_KEYWORD = "domain.displayName.keyword"; |   public static final String DOMAIN_DISPLAY_NAME_KEYWORD = "domain.displayName.keyword"; | ||||||
|   public static final String DATA_MODEL_COLUMNS_NAME_KEYWORD = "dataModel.columns.name.keyword"; |   public static final String DATA_MODEL_COLUMNS_NAME_KEYWORD = "dataModel.columns.name.keyword"; | ||||||
|  | |||||||
| @ -11,11 +11,13 @@ import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_C | |||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; | import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD; | import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; | import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; | ||||||
|  | import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; | import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS; | import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD; | import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG; | import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG; | import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG; | ||||||
|  | import static org.openmetadata.service.search.EntityBuilderConstant.SCHEMA_FIELD_NAMES; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; | import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; | ||||||
| import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; | import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; | ||||||
| 
 | 
 | ||||||
| @ -526,7 +528,9 @@ public class ElasticSearchClient implements SearchClient { | |||||||
|     hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.description").highlighterType(UNIFIED)); |     hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.description").highlighterType(UNIFIED)); | ||||||
|     hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.children.name").highlighterType(UNIFIED)); |     hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.children.name").highlighterType(UNIFIED)); | ||||||
|     SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size); |     SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size); | ||||||
|     searchSourceBuilder.aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD)); |     searchSourceBuilder | ||||||
|  |         .aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD)) | ||||||
|  |         .aggregation(AggregationBuilders.terms(SCHEMA_FIELD_NAMES).field(SCHEMA_FIELD_NAMES)); | ||||||
|     return addAggregation(searchSourceBuilder); |     return addAggregation(searchSourceBuilder); | ||||||
|   } |   } | ||||||
| 
 | 
 | ||||||
| @ -598,6 +602,7 @@ public class ElasticSearchClient implements SearchClient { | |||||||
|     searchSourceBuilder |     searchSourceBuilder | ||||||
|         .aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword")) |         .aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword")) | ||||||
|         .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) |         .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) | ||||||
|  |         .aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES)) | ||||||
|         .aggregation(AggregationBuilders.terms("tableType").field("tableType")); |         .aggregation(AggregationBuilders.terms("tableType").field("tableType")); | ||||||
|     return addAggregation(searchSourceBuilder); |     return addAggregation(searchSourceBuilder); | ||||||
|   } |   } | ||||||
| @ -692,8 +697,9 @@ public class ElasticSearchClient implements SearchClient { | |||||||
|     hb.postTags(POST_TAG); |     hb.postTags(POST_TAG); | ||||||
|     SearchSourceBuilder searchSourceBuilder = |     SearchSourceBuilder searchSourceBuilder = | ||||||
|         new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); |         new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); | ||||||
|     searchSourceBuilder.aggregation( |     searchSourceBuilder | ||||||
|         AggregationBuilders.terms(DATA_MODEL_COLUMNS_NAME_KEYWORD).field(DATA_MODEL_COLUMNS_NAME_KEYWORD)); |         .aggregation(AggregationBuilders.terms(DATA_MODEL_COLUMNS_NAME_KEYWORD).field(DATA_MODEL_COLUMNS_NAME_KEYWORD)) | ||||||
|  |         .aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES)); | ||||||
|     return addAggregation(searchSourceBuilder); |     return addAggregation(searchSourceBuilder); | ||||||
|   } |   } | ||||||
| 
 | 
 | ||||||
| @ -791,7 +797,8 @@ public class ElasticSearchClient implements SearchClient { | |||||||
|     searchSourceBuilder |     searchSourceBuilder | ||||||
|         .aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType")) |         .aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType")) | ||||||
|         .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) |         .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) | ||||||
|         .aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword")); |         .aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword")) | ||||||
|  |         .aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES)); | ||||||
|     return addAggregation(searchSourceBuilder); |     return addAggregation(searchSourceBuilder); | ||||||
|   } |   } | ||||||
| 
 | 
 | ||||||
|  | |||||||
| @ -11,10 +11,13 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD | |||||||
| 
 | 
 | ||||||
| import java.util.ArrayList; | import java.util.ArrayList; | ||||||
| import java.util.HashMap; | import java.util.HashMap; | ||||||
|  | import java.util.HashSet; | ||||||
| import java.util.List; | import java.util.List; | ||||||
| import java.util.Map; | import java.util.Map; | ||||||
|  | import java.util.Set; | ||||||
| import java.util.stream.Collectors; | import java.util.stream.Collectors; | ||||||
| import org.openmetadata.schema.entity.data.Container; | import org.openmetadata.schema.entity.data.Container; | ||||||
|  | import org.openmetadata.schema.type.TagLabel; | ||||||
| import org.openmetadata.service.Entity; | import org.openmetadata.service.Entity; | ||||||
| import org.openmetadata.service.search.ParseTags; | import org.openmetadata.service.search.ParseTags; | ||||||
| import org.openmetadata.service.search.SearchIndexUtils; | import org.openmetadata.service.search.SearchIndexUtils; | ||||||
| @ -36,6 +39,8 @@ public class ContainerIndex implements ColumnIndex { | |||||||
|     List<SearchSuggest> suggest = new ArrayList<>(); |     List<SearchSuggest> suggest = new ArrayList<>(); | ||||||
|     List<SearchSuggest> columnSuggest = new ArrayList<>(); |     List<SearchSuggest> columnSuggest = new ArrayList<>(); | ||||||
|     List<SearchSuggest> serviceSuggest = new ArrayList<>(); |     List<SearchSuggest> serviceSuggest = new ArrayList<>(); | ||||||
|  |     Set<List<TagLabel>> tagsWithChildren = new HashSet<>(); | ||||||
|  |     List<String> columnsWithChildrenName = new ArrayList<>(); | ||||||
|     SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); |     SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); | ||||||
|     suggest.add(SearchSuggest.builder().input(container.getFullyQualifiedName()).weight(5).build()); |     suggest.add(SearchSuggest.builder().input(container.getFullyQualifiedName()).weight(5).build()); | ||||||
|     suggest.add(SearchSuggest.builder().input(container.getName()).weight(10).build()); |     suggest.add(SearchSuggest.builder().input(container.getName()).weight(10).build()); | ||||||
| @ -45,13 +50,21 @@ public class ContainerIndex implements ColumnIndex { | |||||||
| 
 | 
 | ||||||
|       for (FlattenColumn col : cols) { |       for (FlattenColumn col : cols) { | ||||||
|         columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build()); |         columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build()); | ||||||
|  |         columnsWithChildrenName.add(col.getName()); | ||||||
|  |         if (col.getTags() != null) { | ||||||
|  |           tagsWithChildren.add(col.getTags()); | ||||||
|  |         } | ||||||
|       } |       } | ||||||
|  |       doc.put("columnNames", columnsWithChildrenName); | ||||||
|     } |     } | ||||||
|     serviceSuggest.add(SearchSuggest.builder().input(container.getService().getName()).weight(5).build()); |     serviceSuggest.add(SearchSuggest.builder().input(container.getService().getName()).weight(5).build()); | ||||||
|     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.CONTAINER, container)); |     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.CONTAINER, container)); | ||||||
|  |     tagsWithChildren.add(parseTags.getTags()); | ||||||
|  |     List<TagLabel> flattenedTagList = | ||||||
|  |         tagsWithChildren.stream().flatMap(List::stream).collect(ArrayList::new, ArrayList::add, ArrayList::addAll); | ||||||
| 
 | 
 | ||||||
|     doc.put("displayName", container.getDisplayName() != null ? container.getDisplayName() : container.getName()); |     doc.put("displayName", container.getDisplayName() != null ? container.getDisplayName() : container.getName()); | ||||||
|     doc.put("tags", parseTags.getTags()); |     doc.put("tags", flattenedTagList); | ||||||
|     doc.put("tier", parseTags.getTierTag()); |     doc.put("tier", parseTags.getTierTag()); | ||||||
|     doc.put("followers", SearchIndexUtils.parseFollowers(container.getFollowers())); |     doc.put("followers", SearchIndexUtils.parseFollowers(container.getFollowers())); | ||||||
|     doc.put("suggest", suggest); |     doc.put("suggest", suggest); | ||||||
|  | |||||||
| @ -12,16 +12,21 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD | |||||||
| 
 | 
 | ||||||
| import java.util.ArrayList; | import java.util.ArrayList; | ||||||
| import java.util.HashMap; | import java.util.HashMap; | ||||||
|  | import java.util.HashSet; | ||||||
| import java.util.List; | import java.util.List; | ||||||
| import java.util.Map; | import java.util.Map; | ||||||
|  | import java.util.Set; | ||||||
| import java.util.stream.Collectors; | import java.util.stream.Collectors; | ||||||
| import org.openmetadata.schema.entity.data.DashboardDataModel; | import org.openmetadata.schema.entity.data.DashboardDataModel; | ||||||
|  | import org.openmetadata.schema.type.TagLabel; | ||||||
| import org.openmetadata.service.Entity; | import org.openmetadata.service.Entity; | ||||||
|  | import org.openmetadata.service.search.ParseTags; | ||||||
| import org.openmetadata.service.search.SearchIndexUtils; | import org.openmetadata.service.search.SearchIndexUtils; | ||||||
|  | import org.openmetadata.service.search.models.FlattenColumn; | ||||||
| import org.openmetadata.service.search.models.SearchSuggest; | import org.openmetadata.service.search.models.SearchSuggest; | ||||||
| import org.openmetadata.service.util.JsonUtils; | import org.openmetadata.service.util.JsonUtils; | ||||||
| 
 | 
 | ||||||
| public class DashboardDataModelIndex implements SearchIndex { | public class DashboardDataModelIndex implements ColumnIndex { | ||||||
| 
 | 
 | ||||||
|   private static final List<String> excludeFields = List.of("changeDescription"); |   private static final List<String> excludeFields = List.of("changeDescription"); | ||||||
| 
 | 
 | ||||||
| @ -35,8 +40,30 @@ public class DashboardDataModelIndex implements SearchIndex { | |||||||
|     Map<String, Object> doc = JsonUtils.getMap(dashboardDataModel); |     Map<String, Object> doc = JsonUtils.getMap(dashboardDataModel); | ||||||
|     SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); |     SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); | ||||||
|     List<SearchSuggest> suggest = new ArrayList<>(); |     List<SearchSuggest> suggest = new ArrayList<>(); | ||||||
|  |     List<SearchSuggest> columnSuggest = new ArrayList<>(); | ||||||
|     suggest.add(SearchSuggest.builder().input(dashboardDataModel.getName()).weight(10).build()); |     suggest.add(SearchSuggest.builder().input(dashboardDataModel.getName()).weight(10).build()); | ||||||
|     suggest.add(SearchSuggest.builder().input(dashboardDataModel.getFullyQualifiedName()).weight(5).build()); |     suggest.add(SearchSuggest.builder().input(dashboardDataModel.getFullyQualifiedName()).weight(5).build()); | ||||||
|  |     Set<List<TagLabel>> tagsWithChildren = new HashSet<>(); | ||||||
|  |     List<String> columnsWithChildrenName = new ArrayList<>(); | ||||||
|  |     SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); | ||||||
|  |     if (dashboardDataModel.getColumns() != null) { | ||||||
|  |       List<FlattenColumn> cols = new ArrayList<>(); | ||||||
|  |       parseColumns(dashboardDataModel.getColumns(), cols, null); | ||||||
|  |       for (FlattenColumn col : cols) { | ||||||
|  |         columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build()); | ||||||
|  |         columnsWithChildrenName.add(col.getName()); | ||||||
|  |         if (col.getTags() != null) { | ||||||
|  |           tagsWithChildren.add(col.getTags()); | ||||||
|  |         } | ||||||
|  |       } | ||||||
|  |       doc.put("columnNames", columnsWithChildrenName); | ||||||
|  |     } | ||||||
|  |     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.DASHBOARD_DATA_MODEL, dashboardDataModel)); | ||||||
|  |     tagsWithChildren.add(parseTags.getTags()); | ||||||
|  |     List<TagLabel> flattenedTagList = | ||||||
|  |         tagsWithChildren.stream().flatMap(List::stream).collect(ArrayList::new, ArrayList::add, ArrayList::addAll); | ||||||
|  |     doc.put("tags", flattenedTagList); | ||||||
|  |     doc.put("column_suggest", columnSuggest); | ||||||
|     doc.put("suggest", suggest); |     doc.put("suggest", suggest); | ||||||
|     doc.put("entityType", Entity.DASHBOARD_DATA_MODEL); |     doc.put("entityType", Entity.DASHBOARD_DATA_MODEL); | ||||||
|     doc.put( |     doc.put( | ||||||
|  | |||||||
| @ -12,11 +12,14 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD | |||||||
| 
 | 
 | ||||||
| import java.util.ArrayList; | import java.util.ArrayList; | ||||||
| import java.util.HashMap; | import java.util.HashMap; | ||||||
|  | import java.util.HashSet; | ||||||
| import java.util.List; | import java.util.List; | ||||||
| import java.util.Map; | import java.util.Map; | ||||||
|  | import java.util.Set; | ||||||
| import java.util.regex.Pattern; | import java.util.regex.Pattern; | ||||||
| import java.util.stream.Collectors; | import java.util.stream.Collectors; | ||||||
| import org.openmetadata.schema.entity.data.Table; | import org.openmetadata.schema.entity.data.Table; | ||||||
|  | import org.openmetadata.schema.type.TagLabel; | ||||||
| import org.openmetadata.service.Entity; | import org.openmetadata.service.Entity; | ||||||
| import org.openmetadata.service.search.ParseTags; | import org.openmetadata.service.search.ParseTags; | ||||||
| import org.openmetadata.service.search.SearchIndexUtils; | import org.openmetadata.service.search.SearchIndexUtils; | ||||||
| @ -46,6 +49,8 @@ public class TableIndex implements ColumnIndex { | |||||||
|     List<SearchSuggest> schemaSuggest = new ArrayList<>(); |     List<SearchSuggest> schemaSuggest = new ArrayList<>(); | ||||||
|     List<SearchSuggest> databaseSuggest = new ArrayList<>(); |     List<SearchSuggest> databaseSuggest = new ArrayList<>(); | ||||||
|     List<SearchSuggest> serviceSuggest = new ArrayList<>(); |     List<SearchSuggest> serviceSuggest = new ArrayList<>(); | ||||||
|  |     Set<List<TagLabel>> tagsWithChildren = new HashSet<>(); | ||||||
|  |     List<String> columnsWithChildrenName = new ArrayList<>(); | ||||||
|     SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); |     SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); | ||||||
|     if (table.getColumns() != null) { |     if (table.getColumns() != null) { | ||||||
|       List<FlattenColumn> cols = new ArrayList<>(); |       List<FlattenColumn> cols = new ArrayList<>(); | ||||||
| @ -53,15 +58,23 @@ public class TableIndex implements ColumnIndex { | |||||||
| 
 | 
 | ||||||
|       for (FlattenColumn col : cols) { |       for (FlattenColumn col : cols) { | ||||||
|         columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build()); |         columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build()); | ||||||
|  |         columnsWithChildrenName.add(col.getName()); | ||||||
|  |         if (col.getTags() != null) { | ||||||
|  |           tagsWithChildren.add(col.getTags()); | ||||||
|  |         } | ||||||
|       } |       } | ||||||
|  |       doc.put("columnNames", columnsWithChildrenName); | ||||||
|     } |     } | ||||||
|     parseTableSuggest(suggest); |     parseTableSuggest(suggest); | ||||||
|     serviceSuggest.add(SearchSuggest.builder().input(table.getService().getName()).weight(5).build()); |     serviceSuggest.add(SearchSuggest.builder().input(table.getService().getName()).weight(5).build()); | ||||||
|     databaseSuggest.add(SearchSuggest.builder().input(table.getDatabase().getName()).weight(5).build()); |     databaseSuggest.add(SearchSuggest.builder().input(table.getDatabase().getName()).weight(5).build()); | ||||||
|     schemaSuggest.add(SearchSuggest.builder().input(table.getDatabaseSchema().getName()).weight(5).build()); |     schemaSuggest.add(SearchSuggest.builder().input(table.getDatabaseSchema().getName()).weight(5).build()); | ||||||
|     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.TABLE, table)); |     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.TABLE, table)); | ||||||
|  |     tagsWithChildren.add(parseTags.getTags()); | ||||||
|  |     List<TagLabel> flattenedTagList = | ||||||
|  |         tagsWithChildren.stream().flatMap(List::stream).collect(ArrayList::new, ArrayList::add, ArrayList::addAll); | ||||||
|     doc.put("displayName", table.getDisplayName() != null ? table.getDisplayName() : table.getName()); |     doc.put("displayName", table.getDisplayName() != null ? table.getDisplayName() : table.getName()); | ||||||
|     doc.put("tags", parseTags.getTags()); |     doc.put("tags", flattenedTagList); | ||||||
|     doc.put("tier", parseTags.getTierTag()); |     doc.put("tier", parseTags.getTierTag()); | ||||||
|     doc.put("followers", SearchIndexUtils.parseFollowers(table.getFollowers())); |     doc.put("followers", SearchIndexUtils.parseFollowers(table.getFollowers())); | ||||||
|     doc.put( |     doc.put( | ||||||
|  | |||||||
| @ -12,9 +12,11 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD | |||||||
| 
 | 
 | ||||||
| import java.util.ArrayList; | import java.util.ArrayList; | ||||||
| import java.util.HashMap; | import java.util.HashMap; | ||||||
|  | import java.util.HashSet; | ||||||
| import java.util.List; | import java.util.List; | ||||||
| import java.util.Map; | import java.util.Map; | ||||||
| import java.util.Optional; | import java.util.Optional; | ||||||
|  | import java.util.Set; | ||||||
| import java.util.function.Predicate; | import java.util.function.Predicate; | ||||||
| import java.util.stream.Collectors; | import java.util.stream.Collectors; | ||||||
| import org.openmetadata.schema.entity.data.Topic; | import org.openmetadata.schema.entity.data.Topic; | ||||||
| @ -41,6 +43,8 @@ public class TopicIndex implements SearchIndex { | |||||||
|     List<SearchSuggest> suggest = new ArrayList<>(); |     List<SearchSuggest> suggest = new ArrayList<>(); | ||||||
|     List<SearchSuggest> fieldSuggest = new ArrayList<>(); |     List<SearchSuggest> fieldSuggest = new ArrayList<>(); | ||||||
|     List<SearchSuggest> serviceSuggest = new ArrayList<>(); |     List<SearchSuggest> serviceSuggest = new ArrayList<>(); | ||||||
|  |     Set<List<TagLabel>> tagsWithChildren = new HashSet<>(); | ||||||
|  |     List<String> fieldsWithChildrenName = new ArrayList<>(); | ||||||
|     suggest.add(SearchSuggest.builder().input(topic.getFullyQualifiedName()).weight(5).build()); |     suggest.add(SearchSuggest.builder().input(topic.getFullyQualifiedName()).weight(5).build()); | ||||||
|     suggest.add(SearchSuggest.builder().input(topic.getName()).weight(10).build()); |     suggest.add(SearchSuggest.builder().input(topic.getName()).weight(10).build()); | ||||||
|     serviceSuggest.add(SearchSuggest.builder().input(topic.getService().getName()).weight(5).build()); |     serviceSuggest.add(SearchSuggest.builder().input(topic.getService().getName()).weight(5).build()); | ||||||
| @ -54,12 +58,20 @@ public class TopicIndex implements SearchIndex { | |||||||
| 
 | 
 | ||||||
|       for (FlattenSchemaField field : flattenFields) { |       for (FlattenSchemaField field : flattenFields) { | ||||||
|         fieldSuggest.add(SearchSuggest.builder().input(field.getName()).weight(5).build()); |         fieldSuggest.add(SearchSuggest.builder().input(field.getName()).weight(5).build()); | ||||||
|  |         fieldsWithChildrenName.add(field.getName()); | ||||||
|  |         if (field.getTags() != null) { | ||||||
|  |           tagsWithChildren.add(field.getTags()); | ||||||
|  |         } | ||||||
|       } |       } | ||||||
|  |       doc.put("fieldNames", fieldsWithChildrenName); | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.TOPIC, topic)); |     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.TOPIC, topic)); | ||||||
|  |     tagsWithChildren.add(parseTags.getTags()); | ||||||
|  |     List<TagLabel> flattenedTagList = | ||||||
|  |         tagsWithChildren.stream().flatMap(List::stream).collect(ArrayList::new, ArrayList::add, ArrayList::addAll); | ||||||
|     doc.put("displayName", topic.getDisplayName() != null ? topic.getDisplayName() : topic.getName()); |     doc.put("displayName", topic.getDisplayName() != null ? topic.getDisplayName() : topic.getName()); | ||||||
|     doc.put("tags", parseTags.getTags()); |     doc.put("tags", flattenedTagList); | ||||||
|     doc.put("tier", parseTags.getTierTag()); |     doc.put("tier", parseTags.getTierTag()); | ||||||
|     doc.put("followers", SearchIndexUtils.parseFollowers(topic.getFollowers())); |     doc.put("followers", SearchIndexUtils.parseFollowers(topic.getFollowers())); | ||||||
|     doc.put("suggest", suggest); |     doc.put("suggest", suggest); | ||||||
|  | |||||||
| @ -11,11 +11,13 @@ import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_C | |||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; | import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD; | import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; | import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; | ||||||
|  | import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; | import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS; | import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD; | import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG; | import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG; | import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG; | ||||||
|  | import static org.openmetadata.service.search.EntityBuilderConstant.SCHEMA_FIELD_NAMES; | ||||||
| import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; | import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; | ||||||
| import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; | import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; | ||||||
| 
 | 
 | ||||||
| @ -538,7 +540,9 @@ public class OpenSearchClient implements SearchClient { | |||||||
|     hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.description").highlighterType(UNIFIED)); |     hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.description").highlighterType(UNIFIED)); | ||||||
|     hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.children.name").highlighterType(UNIFIED)); |     hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.children.name").highlighterType(UNIFIED)); | ||||||
|     SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size); |     SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size); | ||||||
|     searchSourceBuilder.aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD)); |     searchSourceBuilder | ||||||
|  |         .aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD)) | ||||||
|  |         .aggregation(AggregationBuilders.terms(SCHEMA_FIELD_NAMES).field(SCHEMA_FIELD_NAMES)); | ||||||
|     return addAggregation(searchSourceBuilder); |     return addAggregation(searchSourceBuilder); | ||||||
|   } |   } | ||||||
| 
 | 
 | ||||||
| @ -610,6 +614,7 @@ public class OpenSearchClient implements SearchClient { | |||||||
|     searchSourceBuilder |     searchSourceBuilder | ||||||
|         .aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword")) |         .aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword")) | ||||||
|         .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) |         .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) | ||||||
|  |         .aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES)) | ||||||
|         .aggregation(AggregationBuilders.terms("tableType").field("tableType")); |         .aggregation(AggregationBuilders.terms("tableType").field("tableType")); | ||||||
|     return addAggregation(searchSourceBuilder); |     return addAggregation(searchSourceBuilder); | ||||||
|   } |   } | ||||||
| @ -704,8 +709,9 @@ public class OpenSearchClient implements SearchClient { | |||||||
|     hb.postTags(POST_TAG); |     hb.postTags(POST_TAG); | ||||||
|     SearchSourceBuilder searchSourceBuilder = |     SearchSourceBuilder searchSourceBuilder = | ||||||
|         new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); |         new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); | ||||||
|     searchSourceBuilder.aggregation( |     searchSourceBuilder | ||||||
|         AggregationBuilders.terms(DATA_MODEL_COLUMNS_NAME_KEYWORD).field(DATA_MODEL_COLUMNS_NAME_KEYWORD)); |         .aggregation(AggregationBuilders.terms(DATA_MODEL_COLUMNS_NAME_KEYWORD).field(DATA_MODEL_COLUMNS_NAME_KEYWORD)) | ||||||
|  |         .aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES)); | ||||||
|     return addAggregation(searchSourceBuilder); |     return addAggregation(searchSourceBuilder); | ||||||
|   } |   } | ||||||
| 
 | 
 | ||||||
| @ -803,7 +809,8 @@ public class OpenSearchClient implements SearchClient { | |||||||
|     searchSourceBuilder |     searchSourceBuilder | ||||||
|         .aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType")) |         .aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType")) | ||||||
|         .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) |         .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) | ||||||
|         .aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword")); |         .aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword")) | ||||||
|  |         .aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES)); | ||||||
|     return addAggregation(searchSourceBuilder); |     return addAggregation(searchSourceBuilder); | ||||||
|   } |   } | ||||||
| 
 | 
 | ||||||
|  | |||||||
| @ -438,6 +438,9 @@ | |||||||
|           } |           } | ||||||
|         } |         } | ||||||
|       }, |       }, | ||||||
|  |       "columnNames": { | ||||||
|  |         "type": "keyword" | ||||||
|  |       }, | ||||||
|       "serviceType": { |       "serviceType": { | ||||||
|         "type": "keyword" |         "type": "keyword" | ||||||
|       }, |       }, | ||||||
|  | |||||||
| @ -106,6 +106,9 @@ | |||||||
|           } |           } | ||||||
|         ] |         ] | ||||||
|       }, |       }, | ||||||
|  |       "column_suggest": { | ||||||
|  |         "type": "completion" | ||||||
|  |       }, | ||||||
|       "owner": { |       "owner": { | ||||||
|         "properties": { |         "properties": { | ||||||
|           "id": { |           "id": { | ||||||
| @ -271,6 +274,9 @@ | |||||||
|           } |           } | ||||||
|         } |         } | ||||||
|       }, |       }, | ||||||
|  |       "columnNames": { | ||||||
|  |         "type": "keyword" | ||||||
|  |       }, | ||||||
|       "project": { |       "project": { | ||||||
|         "type": "text", |         "type": "text", | ||||||
|         "fields": { |         "fields": { | ||||||
|  | |||||||
| @ -148,6 +148,9 @@ | |||||||
|           } |           } | ||||||
|         } |         } | ||||||
|       }, |       }, | ||||||
|  |       "columnNames": { | ||||||
|  |         "type": "keyword" | ||||||
|  |       }, | ||||||
|       "databaseSchema": { |       "databaseSchema": { | ||||||
|         "properties": { |         "properties": { | ||||||
|           "id": { |           "id": { | ||||||
|  | |||||||
| @ -284,6 +284,9 @@ | |||||||
|           } |           } | ||||||
|         } |         } | ||||||
|       }, |       }, | ||||||
|  |       "fieldNames": { | ||||||
|  |         "type": "keyword" | ||||||
|  |       }, | ||||||
|       "cleanupPolicies": { |       "cleanupPolicies": { | ||||||
|         "type": "keyword" |         "type": "keyword" | ||||||
|       }, |       }, | ||||||
|  | |||||||
| @ -437,6 +437,9 @@ | |||||||
|       "lifeCycle": { |       "lifeCycle": { | ||||||
|         "type": "object" |         "type": "object" | ||||||
|       }, |       }, | ||||||
|  |       "columnNames": { | ||||||
|  |         "type": "keyword" | ||||||
|  |       }, | ||||||
|       "serviceType": { |       "serviceType": { | ||||||
|         "type": "keyword" |         "type": "keyword" | ||||||
|       }, |       }, | ||||||
|  | |||||||
| @ -95,6 +95,9 @@ | |||||||
|           } |           } | ||||||
|         ] |         ] | ||||||
|       }, |       }, | ||||||
|  |       "column_suggest": { | ||||||
|  |         "type": "completion" | ||||||
|  |       }, | ||||||
|       "version": { |       "version": { | ||||||
|         "type": "float" |         "type": "float" | ||||||
|       }, |       }, | ||||||
| @ -273,6 +276,9 @@ | |||||||
|           } |           } | ||||||
|         } |         } | ||||||
|       }, |       }, | ||||||
|  |       "columnNames": { | ||||||
|  |         "type": "keyword" | ||||||
|  |       }, | ||||||
|       "project": { |       "project": { | ||||||
|         "type": "text", |         "type": "text", | ||||||
|         "fields": { |         "fields": { | ||||||
|  | |||||||
| @ -157,6 +157,9 @@ | |||||||
|           } |           } | ||||||
|         } |         } | ||||||
|       }, |       }, | ||||||
|  |       "columnNames": { | ||||||
|  |         "type": "keyword" | ||||||
|  |       }, | ||||||
|       "databaseSchema": { |       "databaseSchema": { | ||||||
|         "properties": { |         "properties": { | ||||||
|           "id": { |           "id": { | ||||||
|  | |||||||
| @ -237,6 +237,9 @@ | |||||||
|           } |           } | ||||||
|         } |         } | ||||||
|       }, |       }, | ||||||
|  |       "fieldNames": { | ||||||
|  |         "type": "keyword" | ||||||
|  |       }, | ||||||
|       "cleanupPolicies": { |       "cleanupPolicies": { | ||||||
|         "type": "keyword" |         "type": "keyword" | ||||||
|       }, |       }, | ||||||
|  | |||||||
| @ -434,6 +434,9 @@ | |||||||
|           } |           } | ||||||
|         } |         } | ||||||
|       }, |       }, | ||||||
|  |       "columnNames": { | ||||||
|  |         "type": "keyword" | ||||||
|  |       }, | ||||||
|       "serviceType": { |       "serviceType": { | ||||||
|         "type": "keyword" |         "type": "keyword" | ||||||
|       }, |       }, | ||||||
|  | |||||||
| @ -92,6 +92,9 @@ | |||||||
|           } |           } | ||||||
|         ] |         ] | ||||||
|       }, |       }, | ||||||
|  |       "column_suggest": { | ||||||
|  |         "type": "completion" | ||||||
|  |       }, | ||||||
|       "owner": { |       "owner": { | ||||||
|         "properties": { |         "properties": { | ||||||
|           "id": { |           "id": { | ||||||
| @ -258,6 +261,9 @@ | |||||||
|           } |           } | ||||||
|         } |         } | ||||||
|       }, |       }, | ||||||
|  |       "columnNames": { | ||||||
|  |         "type": "keyword" | ||||||
|  |       }, | ||||||
|       "project": { |       "project": { | ||||||
|         "type": "text", |         "type": "text", | ||||||
|         "fields": { |         "fields": { | ||||||
|  | |||||||
| @ -131,6 +131,9 @@ | |||||||
|           } |           } | ||||||
|         } |         } | ||||||
|       }, |       }, | ||||||
|  |       "columnNames": { | ||||||
|  |         "type": "keyword" | ||||||
|  |       }, | ||||||
|       "databaseSchema": { |       "databaseSchema": { | ||||||
|         "properties": { |         "properties": { | ||||||
|           "id": { |           "id": { | ||||||
|  | |||||||
| @ -211,6 +211,9 @@ | |||||||
|           } |           } | ||||||
|         } |         } | ||||||
|       }, |       }, | ||||||
|  |       "fieldNames": { | ||||||
|  |         "type": "keyword" | ||||||
|  |       }, | ||||||
|       "cleanupPolicies": { |       "cleanupPolicies": { | ||||||
|         "type": "keyword" |         "type": "keyword" | ||||||
|       }, |       }, | ||||||
|  | |||||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user
	 07Himank
						07Himank