Fix #3865: Service Connection & Schema Name - Fix ES indexes

catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java

@@ -272,6 +272,9 @@ class TableESIndex extends ElasticSearchIndex {
 
   String database;
 
+  @JsonProperty("database_schema")
+  String databaseSchema;
+
   @JsonProperty("table_type")
   String tableType;
 
@@ -348,14 +351,11 @@ class TableESIndex extends ElasticSearchIndex {
             .tier(parseTags.tierTag);
 
     if (table.getDatabase() != null) {
-      String databaseFQN = table.getDatabase().getName();
-      // TODO fix this code
-      String[] databaseFQNSplit = FullyQualifiedName.split(databaseFQN);
-      if (databaseFQNSplit.length == 2) {
-        tableESIndexBuilder.database(databaseFQNSplit[1]);
-      } else {
-        tableESIndexBuilder.database(databaseFQNSplit[0]);
-      }
+      tableESIndexBuilder.database(table.getDatabase().getName());
+    }
+
+    if (table.getDatabaseSchema() != null) {
+      tableESIndexBuilder.databaseSchema(table.getDatabaseSchema().getName());
     }
 
     if (table.getService() != null) {

catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java

@@ -245,8 +245,8 @@ public class SearchResource {
     hb.field(highlightColumns);
     hb.field(highlightColumnDescriptions);
     SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size);
-    searchSourceBuilder.aggregation(AggregationBuilders.terms("Database").field("database")).highlighter(hb);
-
+    searchSourceBuilder.aggregation(AggregationBuilders.terms("Database").field("database"));
+    searchSourceBuilder.aggregation(AggregationBuilders.terms("DatabaseSchema").field("database_schema"));
     return addAggregation(searchSourceBuilder);
   }
 
@@ -330,6 +330,7 @@ public class SearchResource {
   private SearchSourceBuilder addAggregation(SearchSourceBuilder builder) {
     builder
         .aggregation(AggregationBuilders.terms("Service").field("service_type").size(MAX_AGGREGATE_SIZE))
+        .aggregation(AggregationBuilders.terms("ServiceName").field("service").size(MAX_AGGREGATE_SIZE))
         .aggregation(AggregationBuilders.terms("ServiceCategory").field("service_category").size(MAX_AGGREGATE_SIZE))
         .aggregation(AggregationBuilders.terms("EntityType").field("entity_type"))
         .aggregation(AggregationBuilders.terms("Tier").field("tier"))

catalog-rest-service/src/main/resources/elasticsearch/table_index_mapping.json

@@ -52,6 +52,9 @@
       "database": {
         "type": "keyword"
       },
+      "database_schema": {
+        "type": "keyword"
+      },
       "suggest": {
         "type": "completion"
       },

ingestion/examples/sample_data/dashboards/service.json

@@ -1,12 +1,12 @@
 {
   "name": "sample_superset",
   "serviceType": "Superset",
-  "description": "Supset Service",
-  "connection": {
+  "description": "Superset Service",
+  "dashboardConnection": {
     "config": {
-      "supersetURL": "http://localhost:8088"
+      "supersetURL": "http://localhost:8088",
+      "username": "admin",
+      "password": "admin"
     }
-  },
-  "username": "admin",
-  "password": "admin"
+  }
 }

ingestion/pipelines/metadata_to_es.json

@@ -1,6 +1,7 @@
 {
   "source": {
     "type": "metadata",
+    "serviceName": "openMetadata",
     "config": {
       "include_tables": "true",
       "include_topics": "true",
@@ -18,11 +19,10 @@
       "es_port": 9200
     }
   },
-  "metadata_server": {
-    "type": "metadata-server",
-    "config": {
-      "api_endpoint": "http://localhost:8585/api",
-      "auth_provider_type": "no-auth"
+  "workflowConfig": {
+    "openMetadataServerConfig": {
+      "hostPort": "http://localhost:8585/api",
+      "authProvider": "no-auth"
     }
   }
 }

ingestion/src/metadata/ingestion/models/table_metadata.py

@@ -56,6 +56,7 @@ class TableESDocument(BaseModel):
     table_id: str
     deleted: bool
     database: str
+    database_schema: str
     service: str
     service_type: str
     service_category: str

ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py

@@ -68,6 +68,9 @@ TABLE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
             "database": {
               "type": "keyword"
             },
+            "database_schema": {
+              "type": "keyword"
+            },
             "suggest": {
               "type": "completion"
             },

ingestion/src/metadata/utils/helpers.py

@@ -154,6 +154,7 @@ def get_dashboard_service_or_create(
         return service
     else:
         dashboard_config = {"config": config}
+        print(dashboard_config)
         created_service = metadata.create_or_update(
             CreateDashboardServiceRequest(
                 name=service_name,