Nested columns search fix (#13539)

* nested columns work

* working on nested columns and its nested tags

* worked on nested columns and tags

* removed unneccesary variable

* use only name in arr field

* changed the varable names

* added nested columns logic for container and dashboard datamodel too

* created constants
This commit is contained in:
07Himank 2023-10-13 23:59:05 +05:30 committed by GitHub
parent af9cdc4278
commit 2ef49d52ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 138 additions and 12 deletions

View File

@ -10,6 +10,8 @@ public class EntityBuilderConstant {
public static final String ES_TAG_FQN_FIELD = "tags.tagFQN"; public static final String ES_TAG_FQN_FIELD = "tags.tagFQN";
public static final String COLUMNS_NAME_KEYWORD = "columns.name.keyword"; public static final String COLUMNS_NAME_KEYWORD = "columns.name.keyword";
public static final String FIELD_COLUMN_NAMES = "columnNames";
public static final String SCHEMA_FIELD_NAMES = "fieldNames";
public static final String OWNER_DISPLAY_NAME_KEYWORD = "owner.displayName.keyword"; public static final String OWNER_DISPLAY_NAME_KEYWORD = "owner.displayName.keyword";
public static final String DOMAIN_DISPLAY_NAME_KEYWORD = "domain.displayName.keyword"; public static final String DOMAIN_DISPLAY_NAME_KEYWORD = "domain.displayName.keyword";
public static final String DATA_MODEL_COLUMNS_NAME_KEYWORD = "dataModel.columns.name.keyword"; public static final String DATA_MODEL_COLUMNS_NAME_KEYWORD = "dataModel.columns.name.keyword";

View File

@ -11,11 +11,13 @@ import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_C
import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD; import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES;
import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE;
import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS; import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS;
import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD; import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG; import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG;
import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG; import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG;
import static org.openmetadata.service.search.EntityBuilderConstant.SCHEMA_FIELD_NAMES;
import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED;
import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH;
@ -526,7 +528,9 @@ public class ElasticSearchClient implements SearchClient {
hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.description").highlighterType(UNIFIED)); hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.description").highlighterType(UNIFIED));
hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.children.name").highlighterType(UNIFIED)); hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.children.name").highlighterType(UNIFIED));
SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size); SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size);
searchSourceBuilder.aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD)); searchSourceBuilder
.aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD))
.aggregation(AggregationBuilders.terms(SCHEMA_FIELD_NAMES).field(SCHEMA_FIELD_NAMES));
return addAggregation(searchSourceBuilder); return addAggregation(searchSourceBuilder);
} }
@ -598,6 +602,7 @@ public class ElasticSearchClient implements SearchClient {
searchSourceBuilder searchSourceBuilder
.aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword")) .aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword"))
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES))
.aggregation(AggregationBuilders.terms("tableType").field("tableType")); .aggregation(AggregationBuilders.terms("tableType").field("tableType"));
return addAggregation(searchSourceBuilder); return addAggregation(searchSourceBuilder);
} }
@ -692,8 +697,9 @@ public class ElasticSearchClient implements SearchClient {
hb.postTags(POST_TAG); hb.postTags(POST_TAG);
SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
searchSourceBuilder.aggregation( searchSourceBuilder
AggregationBuilders.terms(DATA_MODEL_COLUMNS_NAME_KEYWORD).field(DATA_MODEL_COLUMNS_NAME_KEYWORD)); .aggregation(AggregationBuilders.terms(DATA_MODEL_COLUMNS_NAME_KEYWORD).field(DATA_MODEL_COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES));
return addAggregation(searchSourceBuilder); return addAggregation(searchSourceBuilder);
} }
@ -791,7 +797,8 @@ public class ElasticSearchClient implements SearchClient {
searchSourceBuilder searchSourceBuilder
.aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType")) .aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType"))
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword")); .aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword"))
.aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES));
return addAggregation(searchSourceBuilder); return addAggregation(searchSourceBuilder);
} }

View File

@ -11,10 +11,13 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.openmetadata.schema.entity.data.Container; import org.openmetadata.schema.entity.data.Container;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.service.Entity; import org.openmetadata.service.Entity;
import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.ParseTags;
import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.SearchIndexUtils;
@ -36,6 +39,8 @@ public class ContainerIndex implements ColumnIndex {
List<SearchSuggest> suggest = new ArrayList<>(); List<SearchSuggest> suggest = new ArrayList<>();
List<SearchSuggest> columnSuggest = new ArrayList<>(); List<SearchSuggest> columnSuggest = new ArrayList<>();
List<SearchSuggest> serviceSuggest = new ArrayList<>(); List<SearchSuggest> serviceSuggest = new ArrayList<>();
Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
List<String> columnsWithChildrenName = new ArrayList<>();
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
suggest.add(SearchSuggest.builder().input(container.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(container.getFullyQualifiedName()).weight(5).build());
suggest.add(SearchSuggest.builder().input(container.getName()).weight(10).build()); suggest.add(SearchSuggest.builder().input(container.getName()).weight(10).build());
@ -45,13 +50,21 @@ public class ContainerIndex implements ColumnIndex {
for (FlattenColumn col : cols) { for (FlattenColumn col : cols) {
columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build()); columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build());
columnsWithChildrenName.add(col.getName());
if (col.getTags() != null) {
tagsWithChildren.add(col.getTags());
}
} }
doc.put("columnNames", columnsWithChildrenName);
} }
serviceSuggest.add(SearchSuggest.builder().input(container.getService().getName()).weight(5).build()); serviceSuggest.add(SearchSuggest.builder().input(container.getService().getName()).weight(5).build());
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.CONTAINER, container)); ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.CONTAINER, container));
tagsWithChildren.add(parseTags.getTags());
List<TagLabel> flattenedTagList =
tagsWithChildren.stream().flatMap(List::stream).collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
doc.put("displayName", container.getDisplayName() != null ? container.getDisplayName() : container.getName()); doc.put("displayName", container.getDisplayName() != null ? container.getDisplayName() : container.getName());
doc.put("tags", parseTags.getTags()); doc.put("tags", flattenedTagList);
doc.put("tier", parseTags.getTierTag()); doc.put("tier", parseTags.getTierTag());
doc.put("followers", SearchIndexUtils.parseFollowers(container.getFollowers())); doc.put("followers", SearchIndexUtils.parseFollowers(container.getFollowers()));
doc.put("suggest", suggest); doc.put("suggest", suggest);

View File

@ -12,16 +12,21 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.openmetadata.schema.entity.data.DashboardDataModel; import org.openmetadata.schema.entity.data.DashboardDataModel;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.service.Entity; import org.openmetadata.service.Entity;
import org.openmetadata.service.search.ParseTags;
import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.SearchIndexUtils;
import org.openmetadata.service.search.models.FlattenColumn;
import org.openmetadata.service.search.models.SearchSuggest; import org.openmetadata.service.search.models.SearchSuggest;
import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.JsonUtils;
public class DashboardDataModelIndex implements SearchIndex { public class DashboardDataModelIndex implements ColumnIndex {
private static final List<String> excludeFields = List.of("changeDescription"); private static final List<String> excludeFields = List.of("changeDescription");
@ -35,8 +40,30 @@ public class DashboardDataModelIndex implements SearchIndex {
Map<String, Object> doc = JsonUtils.getMap(dashboardDataModel); Map<String, Object> doc = JsonUtils.getMap(dashboardDataModel);
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
List<SearchSuggest> suggest = new ArrayList<>(); List<SearchSuggest> suggest = new ArrayList<>();
List<SearchSuggest> columnSuggest = new ArrayList<>();
suggest.add(SearchSuggest.builder().input(dashboardDataModel.getName()).weight(10).build()); suggest.add(SearchSuggest.builder().input(dashboardDataModel.getName()).weight(10).build());
suggest.add(SearchSuggest.builder().input(dashboardDataModel.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(dashboardDataModel.getFullyQualifiedName()).weight(5).build());
Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
List<String> columnsWithChildrenName = new ArrayList<>();
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
if (dashboardDataModel.getColumns() != null) {
List<FlattenColumn> cols = new ArrayList<>();
parseColumns(dashboardDataModel.getColumns(), cols, null);
for (FlattenColumn col : cols) {
columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build());
columnsWithChildrenName.add(col.getName());
if (col.getTags() != null) {
tagsWithChildren.add(col.getTags());
}
}
doc.put("columnNames", columnsWithChildrenName);
}
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.DASHBOARD_DATA_MODEL, dashboardDataModel));
tagsWithChildren.add(parseTags.getTags());
List<TagLabel> flattenedTagList =
tagsWithChildren.stream().flatMap(List::stream).collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
doc.put("tags", flattenedTagList);
doc.put("column_suggest", columnSuggest);
doc.put("suggest", suggest); doc.put("suggest", suggest);
doc.put("entityType", Entity.DASHBOARD_DATA_MODEL); doc.put("entityType", Entity.DASHBOARD_DATA_MODEL);
doc.put( doc.put(

View File

@ -12,11 +12,14 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.service.Entity; import org.openmetadata.service.Entity;
import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.ParseTags;
import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.SearchIndexUtils;
@ -46,6 +49,8 @@ public class TableIndex implements ColumnIndex {
List<SearchSuggest> schemaSuggest = new ArrayList<>(); List<SearchSuggest> schemaSuggest = new ArrayList<>();
List<SearchSuggest> databaseSuggest = new ArrayList<>(); List<SearchSuggest> databaseSuggest = new ArrayList<>();
List<SearchSuggest> serviceSuggest = new ArrayList<>(); List<SearchSuggest> serviceSuggest = new ArrayList<>();
Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
List<String> columnsWithChildrenName = new ArrayList<>();
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
if (table.getColumns() != null) { if (table.getColumns() != null) {
List<FlattenColumn> cols = new ArrayList<>(); List<FlattenColumn> cols = new ArrayList<>();
@ -53,15 +58,23 @@ public class TableIndex implements ColumnIndex {
for (FlattenColumn col : cols) { for (FlattenColumn col : cols) {
columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build()); columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build());
columnsWithChildrenName.add(col.getName());
if (col.getTags() != null) {
tagsWithChildren.add(col.getTags());
}
} }
doc.put("columnNames", columnsWithChildrenName);
} }
parseTableSuggest(suggest); parseTableSuggest(suggest);
serviceSuggest.add(SearchSuggest.builder().input(table.getService().getName()).weight(5).build()); serviceSuggest.add(SearchSuggest.builder().input(table.getService().getName()).weight(5).build());
databaseSuggest.add(SearchSuggest.builder().input(table.getDatabase().getName()).weight(5).build()); databaseSuggest.add(SearchSuggest.builder().input(table.getDatabase().getName()).weight(5).build());
schemaSuggest.add(SearchSuggest.builder().input(table.getDatabaseSchema().getName()).weight(5).build()); schemaSuggest.add(SearchSuggest.builder().input(table.getDatabaseSchema().getName()).weight(5).build());
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.TABLE, table)); ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.TABLE, table));
tagsWithChildren.add(parseTags.getTags());
List<TagLabel> flattenedTagList =
tagsWithChildren.stream().flatMap(List::stream).collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
doc.put("displayName", table.getDisplayName() != null ? table.getDisplayName() : table.getName()); doc.put("displayName", table.getDisplayName() != null ? table.getDisplayName() : table.getName());
doc.put("tags", parseTags.getTags()); doc.put("tags", flattenedTagList);
doc.put("tier", parseTags.getTierTag()); doc.put("tier", parseTags.getTierTag());
doc.put("followers", SearchIndexUtils.parseFollowers(table.getFollowers())); doc.put("followers", SearchIndexUtils.parseFollowers(table.getFollowers()));
doc.put( doc.put(

View File

@ -12,9 +12,11 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate; import java.util.function.Predicate;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.openmetadata.schema.entity.data.Topic; import org.openmetadata.schema.entity.data.Topic;
@ -41,6 +43,8 @@ public class TopicIndex implements SearchIndex {
List<SearchSuggest> suggest = new ArrayList<>(); List<SearchSuggest> suggest = new ArrayList<>();
List<SearchSuggest> fieldSuggest = new ArrayList<>(); List<SearchSuggest> fieldSuggest = new ArrayList<>();
List<SearchSuggest> serviceSuggest = new ArrayList<>(); List<SearchSuggest> serviceSuggest = new ArrayList<>();
Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
List<String> fieldsWithChildrenName = new ArrayList<>();
suggest.add(SearchSuggest.builder().input(topic.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(topic.getFullyQualifiedName()).weight(5).build());
suggest.add(SearchSuggest.builder().input(topic.getName()).weight(10).build()); suggest.add(SearchSuggest.builder().input(topic.getName()).weight(10).build());
serviceSuggest.add(SearchSuggest.builder().input(topic.getService().getName()).weight(5).build()); serviceSuggest.add(SearchSuggest.builder().input(topic.getService().getName()).weight(5).build());
@ -54,12 +58,20 @@ public class TopicIndex implements SearchIndex {
for (FlattenSchemaField field : flattenFields) { for (FlattenSchemaField field : flattenFields) {
fieldSuggest.add(SearchSuggest.builder().input(field.getName()).weight(5).build()); fieldSuggest.add(SearchSuggest.builder().input(field.getName()).weight(5).build());
fieldsWithChildrenName.add(field.getName());
if (field.getTags() != null) {
tagsWithChildren.add(field.getTags());
}
} }
doc.put("fieldNames", fieldsWithChildrenName);
} }
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.TOPIC, topic)); ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.TOPIC, topic));
tagsWithChildren.add(parseTags.getTags());
List<TagLabel> flattenedTagList =
tagsWithChildren.stream().flatMap(List::stream).collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
doc.put("displayName", topic.getDisplayName() != null ? topic.getDisplayName() : topic.getName()); doc.put("displayName", topic.getDisplayName() != null ? topic.getDisplayName() : topic.getName());
doc.put("tags", parseTags.getTags()); doc.put("tags", flattenedTagList);
doc.put("tier", parseTags.getTierTag()); doc.put("tier", parseTags.getTierTag());
doc.put("followers", SearchIndexUtils.parseFollowers(topic.getFollowers())); doc.put("followers", SearchIndexUtils.parseFollowers(topic.getFollowers()));
doc.put("suggest", suggest); doc.put("suggest", suggest);

View File

@ -11,11 +11,13 @@ import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_C
import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD; import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES;
import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE;
import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS; import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS;
import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD; import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG; import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG;
import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG; import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG;
import static org.openmetadata.service.search.EntityBuilderConstant.SCHEMA_FIELD_NAMES;
import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED;
import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH;
@ -538,7 +540,9 @@ public class OpenSearchClient implements SearchClient {
hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.description").highlighterType(UNIFIED)); hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.description").highlighterType(UNIFIED));
hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.children.name").highlighterType(UNIFIED)); hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.children.name").highlighterType(UNIFIED));
SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size); SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size);
searchSourceBuilder.aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD)); searchSourceBuilder
.aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD))
.aggregation(AggregationBuilders.terms(SCHEMA_FIELD_NAMES).field(SCHEMA_FIELD_NAMES));
return addAggregation(searchSourceBuilder); return addAggregation(searchSourceBuilder);
} }
@ -610,6 +614,7 @@ public class OpenSearchClient implements SearchClient {
searchSourceBuilder searchSourceBuilder
.aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword")) .aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword"))
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES))
.aggregation(AggregationBuilders.terms("tableType").field("tableType")); .aggregation(AggregationBuilders.terms("tableType").field("tableType"));
return addAggregation(searchSourceBuilder); return addAggregation(searchSourceBuilder);
} }
@ -704,8 +709,9 @@ public class OpenSearchClient implements SearchClient {
hb.postTags(POST_TAG); hb.postTags(POST_TAG);
SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size); new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
searchSourceBuilder.aggregation( searchSourceBuilder
AggregationBuilders.terms(DATA_MODEL_COLUMNS_NAME_KEYWORD).field(DATA_MODEL_COLUMNS_NAME_KEYWORD)); .aggregation(AggregationBuilders.terms(DATA_MODEL_COLUMNS_NAME_KEYWORD).field(DATA_MODEL_COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES));
return addAggregation(searchSourceBuilder); return addAggregation(searchSourceBuilder);
} }
@ -803,7 +809,8 @@ public class OpenSearchClient implements SearchClient {
searchSourceBuilder searchSourceBuilder
.aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType")) .aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType"))
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD)) .aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword")); .aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword"))
.aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES));
return addAggregation(searchSourceBuilder); return addAggregation(searchSourceBuilder);
} }

View File

@ -438,6 +438,9 @@
} }
} }
}, },
"columnNames": {
"type": "keyword"
},
"serviceType": { "serviceType": {
"type": "keyword" "type": "keyword"
}, },

View File

@ -106,6 +106,9 @@
} }
] ]
}, },
"column_suggest": {
"type": "completion"
},
"owner": { "owner": {
"properties": { "properties": {
"id": { "id": {
@ -271,6 +274,9 @@
} }
} }
}, },
"columnNames": {
"type": "keyword"
},
"project": { "project": {
"type": "text", "type": "text",
"fields": { "fields": {

View File

@ -148,6 +148,9 @@
} }
} }
}, },
"columnNames": {
"type": "keyword"
},
"databaseSchema": { "databaseSchema": {
"properties": { "properties": {
"id": { "id": {

View File

@ -284,6 +284,9 @@
} }
} }
}, },
"fieldNames": {
"type": "keyword"
},
"cleanupPolicies": { "cleanupPolicies": {
"type": "keyword" "type": "keyword"
}, },

View File

@ -437,6 +437,9 @@
"lifeCycle": { "lifeCycle": {
"type": "object" "type": "object"
}, },
"columnNames": {
"type": "keyword"
},
"serviceType": { "serviceType": {
"type": "keyword" "type": "keyword"
}, },

View File

@ -95,6 +95,9 @@
} }
] ]
}, },
"column_suggest": {
"type": "completion"
},
"version": { "version": {
"type": "float" "type": "float"
}, },
@ -273,6 +276,9 @@
} }
} }
}, },
"columnNames": {
"type": "keyword"
},
"project": { "project": {
"type": "text", "type": "text",
"fields": { "fields": {

View File

@ -157,6 +157,9 @@
} }
} }
}, },
"columnNames": {
"type": "keyword"
},
"databaseSchema": { "databaseSchema": {
"properties": { "properties": {
"id": { "id": {

View File

@ -237,6 +237,9 @@
} }
} }
}, },
"fieldNames": {
"type": "keyword"
},
"cleanupPolicies": { "cleanupPolicies": {
"type": "keyword" "type": "keyword"
}, },

View File

@ -434,6 +434,9 @@
} }
} }
}, },
"columnNames": {
"type": "keyword"
},
"serviceType": { "serviceType": {
"type": "keyword" "type": "keyword"
}, },

View File

@ -92,6 +92,9 @@
} }
] ]
}, },
"column_suggest": {
"type": "completion"
},
"owner": { "owner": {
"properties": { "properties": {
"id": { "id": {
@ -258,6 +261,9 @@
} }
} }
}, },
"columnNames": {
"type": "keyword"
},
"project": { "project": {
"type": "text", "type": "text",
"fields": { "fields": {

View File

@ -131,6 +131,9 @@
} }
} }
}, },
"columnNames": {
"type": "keyword"
},
"databaseSchema": { "databaseSchema": {
"properties": { "properties": {
"id": { "id": {

View File

@ -211,6 +211,9 @@
} }
} }
}, },
"fieldNames": {
"type": "keyword"
},
"cleanupPolicies": { "cleanupPolicies": {
"type": "keyword" "type": "keyword"
}, },