yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-08 00:18:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Nested columns search fix (#13539)

Browse Source 
* nested columns work

* working on nested columns and its nested tags

* worked on nested columns and tags

* removed unneccesary variable

* use only name in arr field

* changed the varable names

* added nested columns logic for container and dashboard datamodel too

* created constants
This commit is contained in:
07Himank 2023-10-13 23:59:05 +05:30 committed by GitHub
parent af9cdc4278
commit 2ef49d52ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 138 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java
View File

@ -10,6 +10,8 @@ public class EntityBuilderConstant {
public static final String ES_TAG_FQN_FIELD = "tags.tagFQN";
public static final String COLUMNS_NAME_KEYWORD = "columns.name.keyword";
public static final String FIELD_COLUMN_NAMES = "columnNames";
public static final String SCHEMA_FIELD_NAMES = "fieldNames";
public static final String OWNER_DISPLAY_NAME_KEYWORD = "owner.displayName.keyword";
public static final String DOMAIN_DISPLAY_NAME_KEYWORD = "domain.displayName.keyword";
public static final String DATA_MODEL_COLUMNS_NAME_KEYWORD = "dataModel.columns.name.keyword";

15
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -11,11 +11,13 @@ import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_C
import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES;
import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE;
import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS;
import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG;
import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG;
import static org.openmetadata.service.search.EntityBuilderConstant.SCHEMA_FIELD_NAMES;
import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED;
import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH;
@ -526,7 +528,9 @@ public class ElasticSearchClient implements SearchClient {
hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.description").highlighterType(UNIFIED));
hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.children.name").highlighterType(UNIFIED));
SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size);
searchSourceBuilder.aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD));
searchSourceBuilder
.aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD))
.aggregation(AggregationBuilders.terms(SCHEMA_FIELD_NAMES).field(SCHEMA_FIELD_NAMES));
return addAggregation(searchSourceBuilder);
}
@ -598,6 +602,7 @@ public class ElasticSearchClient implements SearchClient {
searchSourceBuilder
.aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword"))
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES))
.aggregation(AggregationBuilders.terms("tableType").field("tableType"));
return addAggregation(searchSourceBuilder);
}
@ -692,8 +697,9 @@ public class ElasticSearchClient implements SearchClient {
hb.postTags(POST_TAG);
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
searchSourceBuilder.aggregation(
AggregationBuilders.terms(DATA_MODEL_COLUMNS_NAME_KEYWORD).field(DATA_MODEL_COLUMNS_NAME_KEYWORD));
searchSourceBuilder
.aggregation(AggregationBuilders.terms(DATA_MODEL_COLUMNS_NAME_KEYWORD).field(DATA_MODEL_COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES));
return addAggregation(searchSourceBuilder);
}
@ -791,7 +797,8 @@ public class ElasticSearchClient implements SearchClient {
searchSourceBuilder
.aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType"))
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword"));
.aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword"))
.aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES));
return addAggregation(searchSourceBuilder);
}

15
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java
View File

@ -11,10 +11,13 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.openmetadata.schema.entity.data.Container;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.service.Entity;
import org.openmetadata.service.search.ParseTags;
import org.openmetadata.service.search.SearchIndexUtils;
@ -36,6 +39,8 @@ public class ContainerIndex implements ColumnIndex {
List<SearchSuggest> suggest = new ArrayList<>();
List<SearchSuggest> columnSuggest = new ArrayList<>();
List<SearchSuggest> serviceSuggest = new ArrayList<>();
Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
List<String> columnsWithChildrenName = new ArrayList<>();
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
suggest.add(SearchSuggest.builder().input(container.getFullyQualifiedName()).weight(5).build());
suggest.add(SearchSuggest.builder().input(container.getName()).weight(10).build());
@ -45,13 +50,21 @@ public class ContainerIndex implements ColumnIndex {
for (FlattenColumn col : cols) {
columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build());
columnsWithChildrenName.add(col.getName());
if (col.getTags() != null) {
tagsWithChildren.add(col.getTags());
}
}
doc.put("columnNames", columnsWithChildrenName);
}
serviceSuggest.add(SearchSuggest.builder().input(container.getService().getName()).weight(5).build());
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.CONTAINER, container));
tagsWithChildren.add(parseTags.getTags());
List<TagLabel> flattenedTagList =
tagsWithChildren.stream().flatMap(List::stream).collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
doc.put("displayName", container.getDisplayName() != null ? container.getDisplayName() : container.getName());
doc.put("tags", parseTags.getTags());
doc.put("tags", flattenedTagList);
doc.put("tier", parseTags.getTierTag());
doc.put("followers", SearchIndexUtils.parseFollowers(container.getFollowers()));
doc.put("suggest", suggest);

29
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java
View File

@ -12,16 +12,21 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.openmetadata.schema.entity.data.DashboardDataModel;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.service.Entity;
import org.openmetadata.service.search.ParseTags;
import org.openmetadata.service.search.SearchIndexUtils;
import org.openmetadata.service.search.models.FlattenColumn;
import org.openmetadata.service.search.models.SearchSuggest;
import org.openmetadata.service.util.JsonUtils;
public class DashboardDataModelIndex implements SearchIndex {
public class DashboardDataModelIndex implements ColumnIndex {
private static final List<String> excludeFields = List.of("changeDescription");
@ -35,8 +40,30 @@ public class DashboardDataModelIndex implements SearchIndex {
Map<String, Object> doc = JsonUtils.getMap(dashboardDataModel);
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
List<SearchSuggest> suggest = new ArrayList<>();
List<SearchSuggest> columnSuggest = new ArrayList<>();
suggest.add(SearchSuggest.builder().input(dashboardDataModel.getName()).weight(10).build());
suggest.add(SearchSuggest.builder().input(dashboardDataModel.getFullyQualifiedName()).weight(5).build());
Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
List<String> columnsWithChildrenName = new ArrayList<>();
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
if (dashboardDataModel.getColumns() != null) {
List<FlattenColumn> cols = new ArrayList<>();
parseColumns(dashboardDataModel.getColumns(), cols, null);
for (FlattenColumn col : cols) {
columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build());
columnsWithChildrenName.add(col.getName());
if (col.getTags() != null) {
tagsWithChildren.add(col.getTags());
}
}
doc.put("columnNames", columnsWithChildrenName);
}
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.DASHBOARD_DATA_MODEL, dashboardDataModel));
tagsWithChildren.add(parseTags.getTags());
List<TagLabel> flattenedTagList =
tagsWithChildren.stream().flatMap(List::stream).collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
doc.put("tags", flattenedTagList);
doc.put("column_suggest", columnSuggest);
doc.put("suggest", suggest);
doc.put("entityType", Entity.DASHBOARD_DATA_MODEL);
doc.put(

15
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java
View File

@ -12,11 +12,14 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.service.Entity;
import org.openmetadata.service.search.ParseTags;
import org.openmetadata.service.search.SearchIndexUtils;
@ -46,6 +49,8 @@ public class TableIndex implements ColumnIndex {
List<SearchSuggest> schemaSuggest = new ArrayList<>();
List<SearchSuggest> databaseSuggest = new ArrayList<>();
List<SearchSuggest> serviceSuggest = new ArrayList<>();
Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
List<String> columnsWithChildrenName = new ArrayList<>();
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
if (table.getColumns() != null) {
List<FlattenColumn> cols = new ArrayList<>();
@ -53,15 +58,23 @@ public class TableIndex implements ColumnIndex {
for (FlattenColumn col : cols) {
columnSuggest.add(SearchSuggest.builder().input(col.getName()).weight(5).build());
columnsWithChildrenName.add(col.getName());
if (col.getTags() != null) {
tagsWithChildren.add(col.getTags());
}
}
doc.put("columnNames", columnsWithChildrenName);
}
parseTableSuggest(suggest);
serviceSuggest.add(SearchSuggest.builder().input(table.getService().getName()).weight(5).build());
databaseSuggest.add(SearchSuggest.builder().input(table.getDatabase().getName()).weight(5).build());
schemaSuggest.add(SearchSuggest.builder().input(table.getDatabaseSchema().getName()).weight(5).build());
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.TABLE, table));
tagsWithChildren.add(parseTags.getTags());
List<TagLabel> flattenedTagList =
tagsWithChildren.stream().flatMap(List::stream).collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
doc.put("displayName", table.getDisplayName() != null ? table.getDisplayName() : table.getName());
doc.put("tags", parseTags.getTags());
doc.put("tags", flattenedTagList);
doc.put("tier", parseTags.getTierTag());
doc.put("followers", SearchIndexUtils.parseFollowers(table.getFollowers()));
doc.put(

14
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java
View File

@ -12,9 +12,11 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.openmetadata.schema.entity.data.Topic;
@ -41,6 +43,8 @@ public class TopicIndex implements SearchIndex {
List<SearchSuggest> suggest = new ArrayList<>();
List<SearchSuggest> fieldSuggest = new ArrayList<>();
List<SearchSuggest> serviceSuggest = new ArrayList<>();
Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
List<String> fieldsWithChildrenName = new ArrayList<>();
suggest.add(SearchSuggest.builder().input(topic.getFullyQualifiedName()).weight(5).build());
suggest.add(SearchSuggest.builder().input(topic.getName()).weight(10).build());
serviceSuggest.add(SearchSuggest.builder().input(topic.getService().getName()).weight(5).build());
@ -54,12 +58,20 @@ public class TopicIndex implements SearchIndex {
for (FlattenSchemaField field : flattenFields) {
fieldSuggest.add(SearchSuggest.builder().input(field.getName()).weight(5).build());
fieldsWithChildrenName.add(field.getName());
if (field.getTags() != null) {
tagsWithChildren.add(field.getTags());
}
}
doc.put("fieldNames", fieldsWithChildrenName);
}
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.TOPIC, topic));
tagsWithChildren.add(parseTags.getTags());
List<TagLabel> flattenedTagList =
tagsWithChildren.stream().flatMap(List::stream).collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
doc.put("displayName", topic.getDisplayName() != null ? topic.getDisplayName() : topic.getName());
doc.put("tags", parseTags.getTags());
doc.put("tags", flattenedTagList);
doc.put("tier", parseTags.getTierTag());
doc.put("followers", SearchIndexUtils.parseFollowers(topic.getFollowers()));
doc.put("suggest", suggest);

15
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -11,11 +11,13 @@ import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_C
import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES;
import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE;
import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS;
import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG;
import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG;
import static org.openmetadata.service.search.EntityBuilderConstant.SCHEMA_FIELD_NAMES;
import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED;
import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH;
@ -538,7 +540,9 @@ public class OpenSearchClient implements SearchClient {
hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.description").highlighterType(UNIFIED));
hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.children.name").highlighterType(UNIFIED));
SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size);
searchSourceBuilder.aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD));
searchSourceBuilder
.aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD))
.aggregation(AggregationBuilders.terms(SCHEMA_FIELD_NAMES).field(SCHEMA_FIELD_NAMES));
return addAggregation(searchSourceBuilder);
}
@ -610,6 +614,7 @@ public class OpenSearchClient implements SearchClient {
searchSourceBuilder
.aggregation(AggregationBuilders.terms("databaseSchema.name.keyword").field("databaseSchema.name.keyword"))
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES))
.aggregation(AggregationBuilders.terms("tableType").field("tableType"));
return addAggregation(searchSourceBuilder);
}
@ -704,8 +709,9 @@ public class OpenSearchClient implements SearchClient {
hb.postTags(POST_TAG);
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder().query(queryBuilder).highlighter(hb).from(from).size(size);
searchSourceBuilder.aggregation(
AggregationBuilders.terms(DATA_MODEL_COLUMNS_NAME_KEYWORD).field(DATA_MODEL_COLUMNS_NAME_KEYWORD));
searchSourceBuilder
.aggregation(AggregationBuilders.terms(DATA_MODEL_COLUMNS_NAME_KEYWORD).field(DATA_MODEL_COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES));
return addAggregation(searchSourceBuilder);
}
@ -803,7 +809,8 @@ public class OpenSearchClient implements SearchClient {
searchSourceBuilder
.aggregation(AggregationBuilders.terms("dataModelType").field("dataModelType"))
.aggregation(AggregationBuilders.terms(COLUMNS_NAME_KEYWORD).field(COLUMNS_NAME_KEYWORD))
.aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword"));
.aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword"))
.aggregation(AggregationBuilders.terms(FIELD_COLUMN_NAMES).field(FIELD_COLUMN_NAMES));
return addAggregation(searchSourceBuilder);
}

3
openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json
View File

@ -438,6 +438,9 @@
}
}
},
"columnNames": {
"type": "keyword"
},
"serviceType": {
"type": "keyword"
},

6
openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json
View File

@ -106,6 +106,9 @@
}
]
},
"column_suggest": {
"type": "completion"
},
"owner": {
"properties": {
"id": {
@ -271,6 +274,9 @@
}
}
},
"columnNames": {
"type": "keyword"
},
"project": {
"type": "text",
"fields": {

3
openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json
View File

@ -148,6 +148,9 @@
}
}
},
"columnNames": {
"type": "keyword"
},
"databaseSchema": {
"properties": {
"id": {

3
openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json
View File

@ -284,6 +284,9 @@
}
}
},
"fieldNames": {
"type": "keyword"
},
"cleanupPolicies": {
"type": "keyword"
},

3
openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json
View File

@ -437,6 +437,9 @@
"lifeCycle": {
"type": "object"
},
"columnNames": {
"type": "keyword"
},
"serviceType": {
"type": "keyword"
},

6
openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index.json
View File

@ -95,6 +95,9 @@
}
]
},
"column_suggest": {
"type": "completion"
},
"version": {
"type": "float"
},
@ -273,6 +276,9 @@
}
}
},
"columnNames": {
"type": "keyword"
},
"project": {
"type": "text",
"fields": {

3
openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json
View File

@ -157,6 +157,9 @@
}
}
},
"columnNames": {
"type": "keyword"
},
"databaseSchema": {
"properties": {
"id": {

3
openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json
View File

@ -237,6 +237,9 @@
}
}
},
"fieldNames": {
"type": "keyword"
},
"cleanupPolicies": {
"type": "keyword"
},

3
openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json
View File

@ -434,6 +434,9 @@
}
}
},
"columnNames": {
"type": "keyword"
},
"serviceType": {
"type": "keyword"
},

6
openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json
View File

@ -92,6 +92,9 @@
}
]
},
"column_suggest": {
"type": "completion"
},
"owner": {
"properties": {
"id": {
@ -258,6 +261,9 @@
}
}
},
"columnNames": {
"type": "keyword"
},
"project": {
"type": "text",
"fields": {

3
openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json
View File

@ -131,6 +131,9 @@
}
}
},
"columnNames": {
"type": "keyword"
},
"databaseSchema": {
"properties": {
"id": {

3
openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json
View File

@ -211,6 +211,9 @@
}
}
},
"fieldNames": {
"type": "keyword"
},
"cleanupPolicies": {
"type": "keyword"
},