From 4a572713a72cf71d03a8279c9a996553120381aa Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sun, 10 Apr 2022 09:47:36 -0700 Subject: [PATCH] Fix #3865: Service Connection & Schema Name - Fix ES indexes --- .../ElasticSearchIndexDefinition.java | 16 ++++++++-------- .../catalog/resources/search/SearchResource.java | 5 +++-- .../elasticsearch/table_index_mapping.json | 3 +++ .../examples/sample_data/dashboards/service.json | 12 ++++++------ ingestion/pipelines/metadata_to_es.json | 10 +++++----- .../metadata/ingestion/models/table_metadata.py | 1 + .../ingestion/sink/elasticsearch_constants.py | 3 +++ ingestion/src/metadata/utils/helpers.py | 1 + 8 files changed, 30 insertions(+), 21 deletions(-) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java index bcb333cd07c..5ee49d8da07 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java @@ -272,6 +272,9 @@ class TableESIndex extends ElasticSearchIndex { String database; + @JsonProperty("database_schema") + String databaseSchema; + @JsonProperty("table_type") String tableType; @@ -348,14 +351,11 @@ class TableESIndex extends ElasticSearchIndex { .tier(parseTags.tierTag); if (table.getDatabase() != null) { - String databaseFQN = table.getDatabase().getName(); - // TODO fix this code - String[] databaseFQNSplit = FullyQualifiedName.split(databaseFQN); - if (databaseFQNSplit.length == 2) { - tableESIndexBuilder.database(databaseFQNSplit[1]); - } else { - tableESIndexBuilder.database(databaseFQNSplit[0]); - } + tableESIndexBuilder.database(table.getDatabase().getName()); + } + + if (table.getDatabaseSchema() != null) { + tableESIndexBuilder.databaseSchema(table.getDatabaseSchema().getName()); } if (table.getService() != null) { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java index 48080713142..6a180738c2a 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java @@ -245,8 +245,8 @@ public class SearchResource { hb.field(highlightColumns); hb.field(highlightColumnDescriptions); SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size); - searchSourceBuilder.aggregation(AggregationBuilders.terms("Database").field("database")).highlighter(hb); - + searchSourceBuilder.aggregation(AggregationBuilders.terms("Database").field("database")); + searchSourceBuilder.aggregation(AggregationBuilders.terms("DatabaseSchema").field("database_schema")); return addAggregation(searchSourceBuilder); } @@ -330,6 +330,7 @@ public class SearchResource { private SearchSourceBuilder addAggregation(SearchSourceBuilder builder) { builder .aggregation(AggregationBuilders.terms("Service").field("service_type").size(MAX_AGGREGATE_SIZE)) + .aggregation(AggregationBuilders.terms("ServiceName").field("service").size(MAX_AGGREGATE_SIZE)) .aggregation(AggregationBuilders.terms("ServiceCategory").field("service_category").size(MAX_AGGREGATE_SIZE)) .aggregation(AggregationBuilders.terms("EntityType").field("entity_type")) .aggregation(AggregationBuilders.terms("Tier").field("tier")) diff --git a/catalog-rest-service/src/main/resources/elasticsearch/table_index_mapping.json b/catalog-rest-service/src/main/resources/elasticsearch/table_index_mapping.json index 591d89d11a2..fa5838c956d 100644 --- a/catalog-rest-service/src/main/resources/elasticsearch/table_index_mapping.json +++ b/catalog-rest-service/src/main/resources/elasticsearch/table_index_mapping.json @@ -52,6 +52,9 @@ "database": { "type": "keyword" }, + "database_schema": { + "type": "keyword" + }, "suggest": { "type": "completion" }, diff --git a/ingestion/examples/sample_data/dashboards/service.json b/ingestion/examples/sample_data/dashboards/service.json index 4bbee3cbc04..b28ab278102 100644 --- a/ingestion/examples/sample_data/dashboards/service.json +++ b/ingestion/examples/sample_data/dashboards/service.json @@ -1,12 +1,12 @@ { "name": "sample_superset", "serviceType": "Superset", - "description": "Supset Service", - "connection": { + "description": "Superset Service", + "dashboardConnection": { "config": { - "supersetURL": "http://localhost:8088" + "supersetURL": "http://localhost:8088", + "username": "admin", + "password": "admin" } - }, - "username": "admin", - "password": "admin" + } } diff --git a/ingestion/pipelines/metadata_to_es.json b/ingestion/pipelines/metadata_to_es.json index a95361a838f..bb8dd40ea5f 100644 --- a/ingestion/pipelines/metadata_to_es.json +++ b/ingestion/pipelines/metadata_to_es.json @@ -1,6 +1,7 @@ { "source": { "type": "metadata", + "serviceName": "openMetadata", "config": { "include_tables": "true", "include_topics": "true", @@ -18,11 +19,10 @@ "es_port": 9200 } }, - "metadata_server": { - "type": "metadata-server", - "config": { - "api_endpoint": "http://localhost:8585/api", - "auth_provider_type": "no-auth" + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "no-auth" } } } diff --git a/ingestion/src/metadata/ingestion/models/table_metadata.py b/ingestion/src/metadata/ingestion/models/table_metadata.py index c4195ebe3fe..228e26a2f4c 100644 --- a/ingestion/src/metadata/ingestion/models/table_metadata.py +++ b/ingestion/src/metadata/ingestion/models/table_metadata.py @@ -56,6 +56,7 @@ class TableESDocument(BaseModel): table_id: str deleted: bool database: str + database_schema: str service: str service_type: str service_category: str diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py index 559798a265a..e6a29b32bb5 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py @@ -68,6 +68,9 @@ TABLE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "database": { "type": "keyword" }, + "database_schema": { + "type": "keyword" + }, "suggest": { "type": "completion" }, diff --git a/ingestion/src/metadata/utils/helpers.py b/ingestion/src/metadata/utils/helpers.py index c88b02d75f2..008d8dd4e01 100644 --- a/ingestion/src/metadata/utils/helpers.py +++ b/ingestion/src/metadata/utils/helpers.py @@ -154,6 +154,7 @@ def get_dashboard_service_or_create( return service else: dashboard_config = {"config": config} + print(dashboard_config) created_service = metadata.create_or_update( CreateDashboardServiceRequest( name=service_name,