yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-23 16:38:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #5330: Support Advanced search provide multiple field suggestions (#5331)

Browse Source 
* ES suggest

* Fix #5330: Support Advanced search provide multiple field suggestions

* UI : Change table-suggest to metadata-suggest

Co-authored-by: Sachin-chaurasiya <sachinchaurasiyachotey87@gmail.com>
This commit is contained in:
Sriharsha Chintalapani 2022-06-08 00:32:00 -07:00 committed by GitHub
parent 3ee3db8bf5
commit f7438f5469
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 164 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java
View File

@ -235,6 +235,10 @@ class ElasticSearchIndex {
String entityType;
List<ElasticSearchSuggest> suggest;
@JsonProperty("service_suggest")
List<ElasticSearchSuggest> serviceSuggest;
String description;
String tier;
List<String> tags;
@ -326,6 +330,15 @@ class TableESIndex extends ElasticSearchIndex {
@JsonProperty("daily_percentile_rank")
Integer dailyPercentileRank;
@JsonProperty("column_suggest")
List<ElasticSearchSuggest> columnSuggest;
@JsonProperty("schema_suggest")
List<ElasticSearchSuggest> schemaSuggest;
@JsonProperty("database_suggest")
List<ElasticSearchSuggest> databaseSuggest;
public static TableESIndexBuilder builder(Table table, EventType eventType) {
String tableId = table.getId().toString();
String tableName = table.getName();
@ -335,9 +348,15 @@ class TableESIndex extends ElasticSearchIndex {
List<String> columnNames = new ArrayList<>();
List<String> columnDescriptions = new ArrayList<>();
List<ElasticSearchSuggest> suggest = new ArrayList<>();
List<ElasticSearchSuggest> columnSuggest = new ArrayList<>();
List<ElasticSearchSuggest> databaseSuggest = new ArrayList<>();
List<ElasticSearchSuggest> schemaSuggest = new ArrayList<>();
List<ElasticSearchSuggest> serviceSuggest = new ArrayList<>();
suggest.add(ElasticSearchSuggest.builder().input(table.getFullyQualifiedName()).weight(5).build());
suggest.add(ElasticSearchSuggest.builder().input(table.getName()).weight(10).build());
databaseSuggest.add(ElasticSearchSuggest.builder().input(table.getDatabase().getName()).weight(5).build());
schemaSuggest.add(ElasticSearchSuggest.builder().input(table.getDatabaseSchema().getName()).weight(5).build());
serviceSuggest.add(ElasticSearchSuggest.builder().input(table.getService().getName()).weight(5).build());
if (table.getTags() != null) {
table.getTags().forEach(tag -> tags.add(tag.getTagFQN()));
}
@ -352,6 +371,7 @@ class TableESIndex extends ElasticSearchIndex {
}
columnDescriptions.add(col.getDescription());
columnNames.add(col.getName());
columnSuggest.add(ElasticSearchSuggest.builder().input(col.getName()).weight(5).build());
}
}
ParseTags parseTags = new ParseTags(tags);
@ -366,6 +386,10 @@ class TableESIndex extends ElasticSearchIndex {
.lastUpdatedTimestamp(updatedTimestamp)
.fqdn(table.getFullyQualifiedName())
.suggest(suggest)
.columnSuggest(columnSuggest)
.schemaSuggest(schemaSuggest)
.databaseSuggest(databaseSuggest)
.serviceSuggest(serviceSuggest)
.entityType("table")
.columnNames(columnNames)
.columnDescriptions(columnDescriptions)
@ -448,6 +472,8 @@ class TopicESIndex extends ElasticSearchIndex {
public static TopicESIndexBuilder builder(Topic topic, EventType eventType) {
List<String> tags = new ArrayList<>();
List<ElasticSearchSuggest> suggest = new ArrayList<>();
List<ElasticSearchSuggest> serviceSuggest = new ArrayList<>();
serviceSuggest.add(ElasticSearchSuggest.builder().input(topic.getService().getName()).weight(5).build());
suggest.add(ElasticSearchSuggest.builder().input(topic.getFullyQualifiedName()).weight(5).build());
suggest.add(ElasticSearchSuggest.builder().input(topic.getName()).weight(10).build());
@ -468,6 +494,7 @@ class TopicESIndex extends ElasticSearchIndex {
.fqdn(topic.getFullyQualifiedName())
.lastUpdatedTimestamp(updatedTimestamp)
.suggest(suggest)
.serviceSuggest(serviceSuggest)
.serviceCategory("messagingService")
.entityType("topic")
.tags(parseTags.tags)
@ -524,11 +551,17 @@ class DashboardESIndex extends ElasticSearchIndex {
@JsonProperty("daily_percentile_rank")
Integer dailyPercentileRank;
@JsonProperty("chart_suggest")
List<ElasticSearchSuggest> chartSuggest;
public static DashboardESIndexBuilder builder(Dashboard dashboard, EventType eventType) {
List<String> tags = new ArrayList<>();
List<String> chartNames = new ArrayList<>();
List<String> chartDescriptions = new ArrayList<>();
List<ElasticSearchSuggest> suggest = new ArrayList<>();
List<ElasticSearchSuggest> serviceSuggest = new ArrayList<>();
List<ElasticSearchSuggest> chartSuggest = new ArrayList<>();
serviceSuggest.add(ElasticSearchSuggest.builder().input(dashboard.getService().getName()).weight(5).build());
suggest.add(ElasticSearchSuggest.builder().input(dashboard.getFullyQualifiedName()).weight(5).build());
suggest.add(ElasticSearchSuggest.builder().input(dashboard.getDisplayName()).weight(10).build());
Long updatedTimestamp = dashboard.getUpdatedAt();
@ -540,6 +573,7 @@ class DashboardESIndex extends ElasticSearchIndex {
for (EntityReference chart : dashboard.getCharts()) {
chartNames.add(chart.getDisplayName());
chartDescriptions.add(chart.getDescription());
chartSuggest.add(ElasticSearchSuggest.builder().input(chart.getName()).weight(5).build());
}
}
@ -559,6 +593,8 @@ class DashboardESIndex extends ElasticSearchIndex {
.chartDescriptions(chartDescriptions)
.entityType("dashboard")
.suggest(suggest)
.chartSuggest(chartSuggest)
.serviceSuggest(serviceSuggest)
.serviceCategory("dashboardService")
.tags(parseTags.tags)
.tier(parseTags.tierTag);
@ -605,11 +641,17 @@ class PipelineESIndex extends ElasticSearchIndex {
@JsonProperty("task_descriptions")
List<String> taskDescriptions;
@JsonProperty("task_suggest")
List<ElasticSearchSuggest> taskSuggest;
public static PipelineESIndexBuilder builder(Pipeline pipeline, EventType eventType) {
List<String> tags = new ArrayList<>();
List<String> taskNames = new ArrayList<>();
List<String> taskDescriptions = new ArrayList<>();
List<ElasticSearchSuggest> suggest = new ArrayList<>();
List<ElasticSearchSuggest> serviceSuggest = new ArrayList<>();
List<ElasticSearchSuggest> taskSuggest = new ArrayList<>();
serviceSuggest.add(ElasticSearchSuggest.builder().input(pipeline.getService().getName()).weight(5).build());
suggest.add(ElasticSearchSuggest.builder().input(pipeline.getFullyQualifiedName()).weight(5).build());
suggest.add(ElasticSearchSuggest.builder().input(pipeline.getDisplayName()).weight(10).build());
@ -621,6 +663,7 @@ class PipelineESIndex extends ElasticSearchIndex {
for (Task task : pipeline.getTasks()) {
taskNames.add(task.getDisplayName());
taskDescriptions.add(task.getDescription());
taskSuggest.add(ElasticSearchSuggest.builder().input(task.getDisplayName()).weight(5).build());
}
}
@ -642,6 +685,8 @@ class PipelineESIndex extends ElasticSearchIndex {
.taskDescriptions(taskDescriptions)
.entityType("pipeline")
.suggest(suggest)
.taskSuggest(taskSuggest)
.serviceSuggest(serviceSuggest)
.serviceCategory("pipelineService")
.tags(parseTags.tags)
.tier(parseTags.tierTag);

9
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java
View File

@ -197,7 +197,7 @@ public class SearchResource {
@Path("/suggest")
@Operation(
operationId = "getSuggestedEntities",
summary = "Suggest entities",
summary = "Suggest Entities",
tags = "search",
description = "Get suggested entities used for auto-completion.",
responses = {
@ -219,13 +219,14 @@ public class SearchResource {
required = true)
@javax.ws.rs.QueryParam("q")
String query,
@DefaultValue("table_search_index") @javax.ws.rs.QueryParam("index") String index)
@DefaultValue("table_search_index") @javax.ws.rs.QueryParam("index") String index,
@DefaultValue("suggest") @javax.ws.rs.QueryParam("field") String fieldName)
throws IOException {
SearchRequest searchRequest = new SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggest").prefix(query);
CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion(fieldName).prefix(query);
SuggestBuilder suggestBuilder = new SuggestBuilder();
suggestBuilder.addSuggestion("table-suggest", suggestionBuilder);
suggestBuilder.addSuggestion("metadata-suggest", suggestionBuilder);
searchSourceBuilder.suggest(suggestBuilder);
searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS));
searchRequest.source(searchSourceBuilder);

6
catalog-rest-service/src/main/resources/elasticsearch/dashboard_index_mapping.json
View File

@ -117,6 +117,12 @@
"suggest": {
"type": "completion"
},
"chart_suggest": {
"type": "completion"
},
"service_suggest": {
"type": "completion"
},
"monthly_stats":{
"type": "long"
},

3
catalog-rest-service/src/main/resources/elasticsearch/mlmodel_index_mapping.json
View File

@ -80,6 +80,9 @@
"suggest": {
"type": "completion"
},
"service_suggest": {
"type": "completion"
},
"monthly_stats":{
"type": "long"
},

6
catalog-rest-service/src/main/resources/elasticsearch/pipeline_index_mapping.json
View File

@ -116,6 +116,12 @@
},
"suggest": {
"type": "completion"
},
"task_suggest": {
"type": "completion"
},
"service_suggest": {
"type": "completion"
}
}
}

12
catalog-rest-service/src/main/resources/elasticsearch/table_index_mapping.json
View File

@ -123,6 +123,18 @@
"suggest": {
"type": "completion"
},
"column_suggest": {
"type": "completion"
},
"schema_suggest": {
"type": "completion"
},
"database_suggest": {
"type": "completion"
},
"service_suggest": {
"type": "completion"
},
"monthly_stats":{
"type": "long"
},

3
catalog-rest-service/src/main/resources/elasticsearch/topic_index_mapping.json
View File

@ -110,6 +110,9 @@
},
"suggest": {
"type": "completion"
},
"service_suggest": {
"type": "completion"
}
}
}

10
ingestion/src/metadata/ingestion/models/table_metadata.py
View File

@ -52,6 +52,10 @@ class TableESDocument(BaseModel):
entity_type: str = "table"
name: str
suggest: List[dict]
column_suggest: List[dict]
database_suggest: List[dict]
schema_suggest: List[dict]
service_suggest: List[dict]
description: Optional[str] = None
table_type: Optional[str] = None
last_updated_timestamp: Optional[int]
@ -81,6 +85,7 @@ class TopicESDocument(BaseModel):
entity_type: str = "topic"
name: str
suggest: List[dict]
service_suggest: List[dict]
description: Optional[str] = None
last_updated_timestamp: Optional[int]
tags: List[str]
@ -101,6 +106,8 @@ class DashboardESDocument(BaseModel):
entity_type: str = "dashboard"
name: str
suggest: List[dict]
chart_suggest: List[dict]
service_suggest: List[dict]
description: Optional[str] = None
last_updated_timestamp: Optional[int]
chart_names: List[str]
@ -129,6 +136,8 @@ class PipelineESDocument(BaseModel):
entity_type: str = "pipeline"
name: str
suggest: List[dict]
task_suggest: List[dict]
service_suggest: List[dict]
description: Optional[str] = None
last_updated_timestamp: Optional[int]
task_names: List[str]
@ -149,6 +158,7 @@ class MlModelESDocument(BaseModel):
entity_type: str = "mlmodel"
name: str
suggest: List[dict]
service_suggest: List[dict] = None
description: Optional[str] = None
last_updated_timestamp: Optional[int]
algorithm: str

36
ingestion/src/metadata/ingestion/sink/elasticsearch.py
View File

@ -32,10 +32,6 @@ from metadata.generated.schema.entity.data.topic import Topic
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection,
)
from metadata.generated.schema.entity.services.dashboardService import DashboardService
from metadata.generated.schema.entity.services.databaseService import DatabaseService
from metadata.generated.schema.entity.services.messagingService import MessagingService
from metadata.generated.schema.entity.services.pipelineService import PipelineService
from metadata.generated.schema.entity.teams.team import Team
from metadata.generated.schema.entity.teams.user import User
from metadata.generated.schema.type import entityReference
@ -306,6 +302,10 @@ class ElasticsearchSink(Sink[Entity]):
{"input": [table_fqn], "weight": 5},
{"input": [table_name], "weight": 10},
]
column_suggest = []
schema_suggest = []
database_suggest = []
service_suggest = []
column_names = []
column_descriptions = []
tags = set()
@ -341,6 +341,14 @@ class ElasticsearchSink(Sink[Entity]):
deleted=database_entity.service.deleted,
href=database_entity.service.href.__root__,
)
service_suggest.append({"input": [service_entity.name], "weight": 5})
database_suggest.append({"input": [database_entity.name.__root__], "weight": 5})
schema_suggest.append(
{"input": [database_schema_entity.name.__root__], "weight": 5}
)
for column_name in column_names:
column_suggest.append({"input": [column_name], "weight": 5})
table_followers = []
if table.followers:
for follower in table.followers.__root__:
@ -356,6 +364,10 @@ class ElasticsearchSink(Sink[Entity]):
service_type=str(table.serviceType.name),
name=table.name.__root__,
suggest=suggest,
service_suggest=service_suggest,
database_suggest=database_suggest,
schema_suggest=schema_suggest,
column_suggest=column_suggest,
database_schema=str(database_schema_entity.name.__root__),
description=table.description.__root__ if table.description else "",
table_type=table_type,
@ -379,6 +391,7 @@ class ElasticsearchSink(Sink[Entity]):
def _create_topic_es_doc(self, topic: Topic):
topic_fqn = topic.fullyQualifiedName.__root__
topic_name = topic.name
service_suggest = []
suggest = [
{"input": [topic_fqn], "weight": 5},
{"input": [topic_name], "weight": 10},
@ -407,6 +420,7 @@ class ElasticsearchSink(Sink[Entity]):
deleted=topic.service.deleted,
href=topic.service.href.__root__,
)
service_suggest.append({"input": [service_entity.name], "weight": 5})
topic_doc = TopicESDocument(
topic_id=str(topic.id.__root__),
deleted=topic.deleted,
@ -414,6 +428,7 @@ class ElasticsearchSink(Sink[Entity]):
service_type=str(topic.serviceType.name),
name=topic.name.__root__,
suggest=suggest,
service_suggest=service_suggest,
description=topic.description.__root__ if topic.description else "",
last_updated_timestamp=timestamp,
tier=tier,
@ -427,6 +442,8 @@ class ElasticsearchSink(Sink[Entity]):
def _create_dashboard_es_doc(self, dashboard: Dashboard):
dashboard_fqn = dashboard.fullyQualifiedName.__root__
suggest = [{"input": [dashboard.displayName], "weight": 10}]
service_suggest = []
chart_suggest = []
tags = set()
timestamp = dashboard.updatedAt.__root__
dashboard_followers = []
@ -444,6 +461,7 @@ class ElasticsearchSink(Sink[Entity]):
chart_descriptions = []
for chart in charts:
chart_names.append(chart.displayName)
chart_suggest.append({"input": [chart.displayName], "weight": 5})
if chart.description is not None:
chart_descriptions.append(chart.description.__root__)
if len(chart.tags) > 0:
@ -464,6 +482,8 @@ class ElasticsearchSink(Sink[Entity]):
deleted=dashboard.service.deleted,
href=dashboard.service.href.__root__,
)
service_suggest.append({"input": [service_entity.name], "weight": 5})
dashboard_doc = DashboardESDocument(
dashboard_id=str(dashboard.id.__root__),
deleted=dashboard.deleted,
@ -473,6 +493,8 @@ class ElasticsearchSink(Sink[Entity]):
chart_names=chart_names,
chart_descriptions=chart_descriptions,
suggest=suggest,
chart_suggest=chart_suggest,
service_suggest=service_suggest,
description=dashboard.description.__root__ if dashboard.description else "",
last_updated_timestamp=timestamp,
tier=tier,
@ -493,6 +515,8 @@ class ElasticsearchSink(Sink[Entity]):
def _create_pipeline_es_doc(self, pipeline: Pipeline):
pipeline_fqn = pipeline.fullyQualifiedName.__root__
suggest = [{"input": [pipeline.displayName], "weight": 10}]
service_suggest = []
task_suggest = []
tags = set()
timestamp = pipeline.updatedAt.__root__
service_entity = ESEntityReference(
@ -509,6 +533,7 @@ class ElasticsearchSink(Sink[Entity]):
deleted=pipeline.service.deleted,
href=pipeline.service.href.__root__,
)
service_suggest.append({"input": [service_entity.name], "weight": 5})
pipeline_followers = []
if pipeline.followers:
@ -525,6 +550,7 @@ class ElasticsearchSink(Sink[Entity]):
task_descriptions = []
for task in tasks:
task_names.append(task.displayName)
task_suggest.append({"input": [task.displayName], "weight": 5})
if task.description:
task_descriptions.append(task.description.__root__)
if tags in task and len(task.tags) > 0:
@ -540,6 +566,8 @@ class ElasticsearchSink(Sink[Entity]):
task_names=task_names,
task_descriptions=task_descriptions,
suggest=suggest,
task_suggest=task_suggest,
service_suggest=service_suggest,
description=pipeline.description.__root__ if pipeline.description else "",
last_updated_timestamp=timestamp,
tier=tier,

32
ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py
View File

@ -139,6 +139,18 @@ TABLE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
"suggest": {
"type": "completion"
},
"column_suggest": {
"type": "completion"
},
"schema_suggest": {
"type": "completion"
},
"database_suggest": {
"type": "completion"
},
"service_suggest": {
"type": "completion"
},
"monthly_stats":{
"type": "long"
},
@ -278,6 +290,9 @@ TOPIC_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
},
"suggest": {
"type": "completion"
},
"service_suggest": {
"type": "completion"
}
}
}
@ -406,6 +421,12 @@ DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
},
"suggest": {
"type": "completion"
},
"chart_suggest": {
"type": "completion"
},
"service_suggest": {
"type": "completion"
},
"monthly_stats":{
"type": "long"
@ -552,7 +573,13 @@ PIPELINE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
},
"suggest": {
"type": "completion"
}
},
"task_suggest": {
"type": "completion"
},
"service_suggest": {
"type": "completion"
}
}
}
}
@ -944,6 +971,9 @@ MLMODEL_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
"suggest": {
"type": "completion"
},
"service_suggest": {
"type": "completion"
},
"monthly_stats": {
"type": "long"
},

2
openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/NodeSuggestions.component.tsx
View File

@ -60,7 +60,7 @@ const NodeSuggestions: FC<EntitySuggestionProps> = ({
.then((res: AxiosResponse) => {
if (res.data) {
setData(
formatDataResponse(res.data.suggest['table-suggest'][0].options)
formatDataResponse(res.data.suggest['metadata-suggest'][0].options)
);
} else {
throw jsonData['api-error-messages']['unexpected-server-response'];

2
openmetadata-ui/src/main/resources/ui/src/components/Modals/ReviewerModal/ReviewerModal.component.tsx
View File

@ -75,7 +75,7 @@ const ReviewerModal = ({
getSuggestions(searchText, SearchIndex.USER)
.then((res: AxiosResponse) => {
const data = formatUsersResponse(
res.data.suggest['table-suggest'][0].options
res.data.suggest['metadata-suggest'][0].options
);
setOptions(data);
})

2
openmetadata-ui/src/main/resources/ui/src/components/TeamsSelectable/TeamsSelectable.test.tsx
View File

@ -45,7 +45,7 @@ jest.mock('../../axiosAPIs/miscAPI', () => ({
Promise.resolve({
data: {
suggest: {
'table-suggest': [{ options: [] }],
'metadata-suggest': [{ options: [] }],
},
},
})

2
openmetadata-ui/src/main/resources/ui/src/components/TeamsSelectable/TeamsSelectable.tsx
View File

@ -42,7 +42,7 @@ const TeamsSelectable = ({ onSelectionChange }: Props) => {
if (text) {
getSuggestedTeams(text).then((res) => {
const teams: UserTeams[] = formatTeamsResponse(
res.data.suggest['table-suggest'][0].options
res.data.suggest['metadata-suggest'][0].options
);
const options = teams.map((team) => ({
label: getEntityName(team),

4
openmetadata-ui/src/main/resources/ui/src/components/app-bar/Suggestions.tsx
View File

@ -253,8 +253,8 @@ const Suggestions = ({ searchText, isOpen, setIsOpen }: SuggestionProp) => {
getSuggestions(searchText)
.then((res: AxiosResponse) => {
if (res.data) {
setOptions(res.data.suggest['table-suggest'][0].options);
setSuggestions(res.data.suggest['table-suggest'][0].options);
setOptions(res.data.suggest['metadata-suggest'][0].options);
setSuggestions(res.data.suggest['metadata-suggest'][0].options);
setIsOpen(true);
} else {
throw jsonData['api-error-messages']['unexpected-server-response'];

4
openmetadata-ui/src/main/resources/ui/src/utils/FeedUtils.tsx
View File

@ -166,7 +166,7 @@ export async function suggestions(searchTerm: string, mentionChar: string) {
});
} else {
const data = await getUserSuggestions(searchTerm);
const hits = data.data.suggest['table-suggest'][0]['options'];
const hits = data.data.suggest['metadata-suggest'][0]['options'];
// eslint-disable-next-line @typescript-eslint/no-explicit-any
atValues = hits.map((hit: any) => {
const entityType = hit._source.entity_type;
@ -202,7 +202,7 @@ export async function suggestions(searchTerm: string, mentionChar: string) {
});
} else {
const data = await getSuggestions(searchTerm);
const hits = data.data.suggest['table-suggest'][0]['options'];
const hits = data.data.suggest['metadata-suggest'][0]['options'];
// eslint-disable-next-line @typescript-eslint/no-explicit-any
hashValues = hits.map((hit: any) => {
const entityType = hit._source.entity_type;

4
openmetadata-ui/src/main/resources/ui/src/utils/UserDataUtils.ts
View File

@ -188,13 +188,13 @@ export const suggestFormattedUsersAndTeams = (
const users =
resUsers.status === SettledStatus.FULFILLED
? formatUsersResponse(
resUsers.value.data.suggest['table-suggest'][0].options
resUsers.value.data.suggest['metadata-suggest'][0].options
)
: [];
const teams =
resTeams.status === SettledStatus.FULFILLED
? formatTeamsResponse(
resTeams.value.data.suggest['table-suggest'][0].options
resTeams.value.data.suggest['metadata-suggest'][0].options
)
: [];
resolve({ users, teams });