From 9b35547fa5ce7fe2e3178ead7debd1e65be37132 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Mon, 19 Sep 2022 17:22:23 -0700 Subject: [PATCH] Fix #7562: Fix search suggest for soft-deleted entities (#7563) --- .../metadata/ingestion/sink/elasticsearch.py | 5 +- .../dashboard_search_index_mapping.py | 17 +- .../glossary_term_search_index_mapping.py | 19 +- .../mlmodel_search_index_mapping.py | 17 +- .../pipeline_search_index_mapping.py | 15 +- .../table_search_index_mapping.py | 767 +++++++++--------- .../tag_search_index_mapping.py | 11 +- .../team_search_index_mapping.py | 15 +- .../topic_search_index_mapping.py | 15 +- .../user_search_index_mapping.py | 17 +- .../ElasticSearchIndexDefinition.java | 2 + .../resources/search/SearchResource.java | 10 +- .../dashboard_index_mapping.json | 17 +- .../elasticsearch/glossary_index_mapping.json | 19 +- .../elasticsearch/mlmodel_index_mapping.json | 17 +- .../elasticsearch/pipeline_index_mapping.json | 15 +- .../elasticsearch/table_index_mapping.json | 21 +- .../elasticsearch/tag_index_mapping.json | 11 +- .../elasticsearch/team_index_mapping.json | 15 +- .../elasticsearch/topic_index_mapping.json | 15 +- .../elasticsearch/user_index_mapping.json | 8 +- 21 files changed, 597 insertions(+), 451 deletions(-) diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch.py b/ingestion/src/metadata/ingestion/sink/elasticsearch.py index bd42b6b0899..22a354123e6 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch.py @@ -389,7 +389,10 @@ class ElasticsearchSink(Sink[Entity]): service_suggest.append({"input": [table.service.name], "weight": 5}) database_suggest.append({"input": [database_entity.name.__root__], "weight": 5}) schema_suggest.append( - {"input": [database_schema_entity.name.__root__], "weight": 5} + { + "input": [database_schema_entity.name.__root__], + "weight": 5, + } ) self._parse_columns( table.columns, None, column_names, column_descriptions, tags diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/dashboard_search_index_mapping.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/dashboard_search_index_mapping.py index 3972069ddb9..01626238db5 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/dashboard_search_index_mapping.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/dashboard_search_index_mapping.py @@ -89,7 +89,7 @@ DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -126,7 +126,7 @@ DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -163,7 +163,7 @@ DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -205,7 +205,7 @@ DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( } }, "deleted": { - "type": "boolean" + "type": "text" }, "followers": { "type": "keyword" @@ -255,7 +255,14 @@ DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "keyword" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] }, "chart_suggest": { "type": "completion" diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/glossary_term_search_index_mapping.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/glossary_term_search_index_mapping.py index 60807a3ec77..5c7033d6598 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/glossary_term_search_index_mapping.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/glossary_term_search_index_mapping.py @@ -78,7 +78,7 @@ GLOSSARY_TERM_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -115,7 +115,7 @@ GLOSSARY_TERM_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -152,7 +152,7 @@ GLOSSARY_TERM_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -189,7 +189,7 @@ GLOSSARY_TERM_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -219,13 +219,20 @@ GLOSSARY_TERM_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( } }, "deleted": { - "type": "boolean" + "type": "text" }, "status": { "type": "text" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] } } } diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/mlmodel_search_index_mapping.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/mlmodel_search_index_mapping.py index 3a71dc0cbb3..998e334b403 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/mlmodel_search_index_mapping.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/mlmodel_search_index_mapping.py @@ -128,7 +128,7 @@ MLMODEL_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -178,7 +178,7 @@ MLMODEL_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -215,7 +215,7 @@ MLMODEL_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -257,7 +257,7 @@ MLMODEL_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( } }, "deleted": { - "type": "boolean" + "type": "text" }, "followers": { "type": "keyword" @@ -307,7 +307,14 @@ MLMODEL_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "keyword" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] }, "service_suggest": { "type": "completion" diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/pipeline_search_index_mapping.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/pipeline_search_index_mapping.py index 8f0987dfc6b..d35d8843ae6 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/pipeline_search_index_mapping.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/pipeline_search_index_mapping.py @@ -108,7 +108,7 @@ PIPELINE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -145,7 +145,7 @@ PIPELINE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -153,7 +153,7 @@ PIPELINE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( } }, "deleted": { - "type": "boolean" + "type": "text" }, "followers": { "type": "keyword" @@ -203,7 +203,14 @@ PIPELINE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "keyword" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] }, "task_suggest": { "type": "completion" diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/table_search_index_mapping.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/table_search_index_mapping.py index 4fe048311df..f75805a3103 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/table_search_index_mapping.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/table_search_index_mapping.py @@ -14,379 +14,400 @@ import textwrap TABLE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( """ { - "mappings":{ - "properties": { - "id": { - "type": "text" - }, - "name": { - "type":"text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type":"text" - }, - "displayName": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "description": { - "type": "text", - "index_options": "docs", - "norms": false - }, - "version": { - "type": "float" - }, - "updatedAt": { - "type": "date", - "format": "epoch_second" - }, - "updatedBy": { - "type": "text" - }, - "href": { - "type": "text" - }, - "columns": { - "properties": { - "name": { - "type": "text" - }, - "dataType": { - "type": "text" - }, - "dataTypeDisplay": { - "type": "text" - }, - "description": { - "type": "text", - "index_options": "docs", - "norms": false - }, - "fullyQualifiedName": { - "type": "text" - }, - "tags": { - "properties": { - "tagFQN": { - "type": "keyword" - }, - "labelType": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "source": { - "type": "keyword" - }, - "state": { - "type": "keyword" - } - } - }, - "ordinalPosition": { - "type": "integer" - } - } - }, - "databaseSchema": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "text" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } - }, - "database": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } - }, - "service": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } - }, - "owner": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } - }, - "location": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } - }, - "usageSummary": { - "properties": { - "dailyStats": { - "properties": { - "count": { - "type": "long" - }, - "percentileRank": { - "type": "long" - } - } - }, - "weeklyStats": { - "properties": { - "count": { - "type": "long" - }, - "percentileRank": { - "type": "long" - } - } - }, - "monthlyStats": { - "properties": { - "count": { - "type": "long" - }, - "percentileRank": { - "type": "long" - } - } - } - } - }, - "deleted": { - "type": "boolean" - }, - "followers": { - "type": "keyword" - }, - "tier": { - "properties": { - "tagFQN": { - "type": "keyword" - }, - "labelType": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "source": { - "type": "keyword" - }, - "state": { - "type": "keyword" - } - } - }, - "tags": { - "properties": { - "tagFQN": { - "type": "keyword" - }, - "labelType": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "source": { - "type": "keyword" - }, - "state": { - "type": "keyword" - } - } - }, - "serviceType": { - "type": "keyword" - }, - "entityType": { - "type": "keyword" - }, - "suggest": { - "type": "completion" - }, - "column_suggest": { - "type": "completion" - }, - "schema_suggest": { - "type": "completion" - }, - "database_suggest": { - "type": "completion" - }, - "service_suggest": { - "type": "completion" - } - } + "settings": { + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } } - } + } + }, + "mappings": { + "properties": { + "id": { + "type": "text" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text", + "index_options": "docs", + "norms": false + }, + "version": { + "type": "float" + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "columns": { + "properties": { + "name": { + "type": "text" + }, + "dataType": { + "type": "text" + }, + "dataTypeDisplay": { + "type": "text" + }, + "description": { + "type": "text", + "index_options": "docs", + "norms": false + }, + "fullyQualifiedName": { + "type": "text" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "ordinalPosition": { + "type": "integer" + } + } + }, + "databaseSchema": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "text" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "database": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "service": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "owner": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "location": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "usageSummary": { + "properties": { + "dailyStats": { + "properties": { + "count": { + "type": "long" + }, + "percentileRank": { + "type": "long" + } + } + }, + "weeklyStats": { + "properties": { + "count": { + "type": "long" + }, + "percentileRank": { + "type": "long" + } + } + }, + "monthlyStats": { + "properties": { + "count": { + "type": "long" + }, + "percentileRank": { + "type": "long" + } + } + } + } + }, + "deleted": { + "type": "text" + }, + "followers": { + "type": "keyword" + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "serviceType": { + "type": "keyword" + }, + "entityType": { + "type": "keyword" + }, + "suggest": { + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] + }, + "column_suggest": { + "type": "completion" + }, + "schema_suggest": { + "type": "completion" + }, + "database_suggest": { + "type": "completion" + }, + "service_suggest": { + "type": "completion" + } + } + } +} """ ) diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/tag_search_index_mapping.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/tag_search_index_mapping.py index 9685a644368..ba2b28bc478 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/tag_search_index_mapping.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/tag_search_index_mapping.py @@ -51,13 +51,20 @@ TAG_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "deprecated": { "type": "boolean" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] } } } diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/team_search_index_mapping.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/team_search_index_mapping.py index bc45ea9b511..5b9f37e44b7 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/team_search_index_mapping.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/team_search_index_mapping.py @@ -75,7 +75,7 @@ TEAM_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -112,7 +112,7 @@ TEAM_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -123,13 +123,20 @@ TEAM_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "entityType": { "type": "keyword" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] } } } diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/topic_search_index_mapping.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/topic_search_index_mapping.py index 55b9f62a11e..075e1019ea9 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/topic_search_index_mapping.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/topic_search_index_mapping.py @@ -104,7 +104,7 @@ TOPIC_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -141,7 +141,7 @@ TOPIC_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -149,7 +149,7 @@ TOPIC_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( } }, "deleted": { - "type": "boolean" + "type": "text" }, "followers": { "type": "keyword" @@ -199,7 +199,14 @@ TOPIC_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "keyword" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] }, "service_suggest": { "type": "completion" diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/user_search_index_mapping.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/user_search_index_mapping.py index b94268f6d56..114bdce585c 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/user_search_index_mapping.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/user_search_index_mapping.py @@ -78,7 +78,7 @@ USER_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -115,7 +115,7 @@ USER_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -152,7 +152,7 @@ USER_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -160,13 +160,20 @@ USER_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( } }, "deleted": { - "type": "boolean" + "type": "text" }, "entityType": { "type": "keyword" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexDefinition.java b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexDefinition.java index 848ffc02f54..1498012f5c3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexDefinition.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/elasticsearch/ElasticSearchIndexDefinition.java @@ -1,5 +1,6 @@ package org.openmetadata.service.elasticsearch; +import com.fasterxml.jackson.annotation.JsonInclude; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -177,6 +178,7 @@ public class ElasticSearchIndexDefinition { } } +@JsonInclude(JsonInclude.Include.NON_NULL) @Jacksonized @Getter @Builder diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java index 53de4b71067..6d152b74d52 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java @@ -25,6 +25,7 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import java.io.IOException; +import java.util.Collections; import java.util.concurrent.TimeUnit; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; @@ -53,6 +54,7 @@ import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.SuggestBuilders; import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; +import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext; import org.openmetadata.schema.api.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.service.util.ElasticSearchClientUtils; @@ -225,11 +227,17 @@ public class SearchResource { @javax.ws.rs.QueryParam("q") String query, @DefaultValue("table_search_index") @javax.ws.rs.QueryParam("index") String index, - @DefaultValue("suggest") @javax.ws.rs.QueryParam("field") String fieldName) + @DefaultValue("suggest") @javax.ws.rs.QueryParam("field") String fieldName, + @DefaultValue("false") @javax.ws.rs.QueryParam("deleted") String deleted) throws IOException { SearchRequest searchRequest = new SearchRequest(index); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion(fieldName).prefix(query); + if (fieldName.equalsIgnoreCase("suggest")) { + suggestionBuilder.contexts( + Collections.singletonMap( + "deleted", Collections.singletonList(CategoryQueryContext.builder().setCategory(deleted).build()))); + } SuggestBuilder suggestBuilder = new SuggestBuilder(); suggestBuilder.addSuggestion("metadata-suggest", suggestionBuilder); searchSourceBuilder.suggest(suggestBuilder); diff --git a/openmetadata-service/src/main/resources/elasticsearch/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/dashboard_index_mapping.json index 58d5a8c0e73..f874c7cf9cd 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/dashboard_index_mapping.json @@ -88,7 +88,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -125,7 +125,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -162,7 +162,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -204,7 +204,7 @@ } }, "deleted": { - "type": "boolean" + "type": "text" }, "followers": { "type": "keyword" @@ -238,7 +238,14 @@ "type": "keyword" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] }, "chart_suggest": { "type": "completion" diff --git a/openmetadata-service/src/main/resources/elasticsearch/glossary_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/glossary_index_mapping.json index 18f203dfcd6..545acfb66d8 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/glossary_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/glossary_index_mapping.json @@ -88,7 +88,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -125,7 +125,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -162,7 +162,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -199,7 +199,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -229,13 +229,20 @@ } }, "deleted": { - "type": "boolean" + "type": "text" }, "status": { "type": "text" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] } } } diff --git a/openmetadata-service/src/main/resources/elasticsearch/mlmodel_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/mlmodel_index_mapping.json index a70a82216ea..311100a2d96 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/mlmodel_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/mlmodel_index_mapping.json @@ -127,7 +127,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -177,7 +177,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -214,7 +214,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -256,7 +256,7 @@ } }, "deleted": { - "type": "boolean" + "type": "text" }, "followers": { "type": "keyword" @@ -290,7 +290,14 @@ "type": "keyword" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] }, "service_suggest": { "type": "completion" diff --git a/openmetadata-service/src/main/resources/elasticsearch/pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/pipeline_index_mapping.json index b5e6d03662b..e8b5e5bb123 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/pipeline_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/pipeline_index_mapping.json @@ -107,7 +107,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -144,7 +144,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -152,7 +152,7 @@ } }, "deleted": { - "type": "boolean" + "type": "text" }, "followers": { "type": "keyword" @@ -186,7 +186,14 @@ "type": "keyword" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] }, "task_suggest": { "type": "completion" diff --git a/openmetadata-service/src/main/resources/elasticsearch/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/table_index_mapping.json index f6c27f8a8c8..4c8062575d9 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/table_index_mapping.json @@ -130,7 +130,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -167,7 +167,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -204,7 +204,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -241,7 +241,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -278,7 +278,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -320,7 +320,7 @@ } }, "deleted": { - "type": "boolean" + "type": "text" }, "followers": { "type": "keyword" @@ -370,7 +370,14 @@ "type": "keyword" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] }, "column_suggest": { "type": "completion" diff --git a/openmetadata-service/src/main/resources/elasticsearch/tag_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/tag_index_mapping.json index 17cf7a093cf..de780699996 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/tag_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/tag_index_mapping.json @@ -59,10 +59,17 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] } } } diff --git a/openmetadata-service/src/main/resources/elasticsearch/team_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/team_index_mapping.json index 31d700d1b38..20c3baf60c4 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/team_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/team_index_mapping.json @@ -65,7 +65,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -102,7 +102,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -113,13 +113,20 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "entityType": { "type": "keyword" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] } } } diff --git a/openmetadata-service/src/main/resources/elasticsearch/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/topic_index_mapping.json index aa9b3c4d934..2b7053413e2 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/topic_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/topic_index_mapping.json @@ -103,7 +103,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -140,7 +140,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -148,7 +148,7 @@ } }, "deleted": { - "type": "boolean" + "type": "text" }, "followers": { "type": "keyword" @@ -182,7 +182,14 @@ "type": "keyword" }, "suggest": { - "type": "completion" + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] }, "service_suggest": { "type": "completion" diff --git a/openmetadata-service/src/main/resources/elasticsearch/user_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/user_index_mapping.json index aeac7f1a55a..5bc841411eb 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/user_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/user_index_mapping.json @@ -74,7 +74,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -111,7 +111,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -148,7 +148,7 @@ "type": "text" }, "deleted": { - "type": "boolean" + "type": "text" }, "href": { "type": "text" @@ -156,7 +156,7 @@ } }, "deleted": { - "type": "boolean" + "type": "text" }, "entityType": { "type": "keyword"