Fixes #1508 Add serviceType and service reference for Chart, Dashboard, Database, Pipeline, Table, and Topic (#1510)

Co-authored-by: sureshms <suresh@getcollate.io>

.idea/encodings.xml

@@ -5,5 +5,6 @@
     <file url="file://$PROJECT_DIR$/catalog-rest-service/src/main/resources" charset="UTF-8" />
     <file url="file://$PROJECT_DIR$/catalog-rest-service/src/main/resources/json/data" charset="UTF-8" />
     <file url="file://$PROJECT_DIR$/common/src/main/java" charset="UTF-8" />
+    <file url="file://$PROJECT_DIR$/openmetadata-ui/src/main/resources/ui/dist" charset="UTF-8" />
   </component>
 </project>

.idea/misc.xml

@@ -8,5 +8,5 @@
       </list>
     </option>
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="11" languageLevel="JDK_11"  project-jdk-type="JavaSDK" />
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_11" project-jdk-name="11" project-jdk-type="JavaSDK" />
 </project>

catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogExceptionMessage.java

@@ -56,4 +56,8 @@ public final class CatalogExceptionMessage {
   public static String entityVersionNotFound(String entity, String id, Double version) {
     return String.format("%s instance for %s and version %s not found", StringUtils.capitalize(entity), id, version);
   }
+
+  public static String invalidServiceEntity(String serviceEntity, String entity) {
+    return String.format("Invalid service entity type %s for %s", serviceEntity, entity);
+  }
 }

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java

@@ -18,7 +18,9 @@ import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.entity.data.Chart;
 import org.openmetadata.catalog.entity.services.DashboardService;
+import org.openmetadata.catalog.exception.CatalogExceptionMessage;
 import org.openmetadata.catalog.exception.EntityNotFoundException;
+import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface;
 import org.openmetadata.catalog.resources.charts.ChartResource;
 import org.openmetadata.catalog.type.ChangeDescription;
 import org.openmetadata.catalog.type.EntityReference;
@@ -33,11 +35,11 @@ import java.net.URI;
 import java.text.ParseException;
 import java.util.Date;
 import java.util.List;
-import java.util.Objects;
 import java.util.UUID;
 
 import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound;
 
+
 public class ChartRepository extends EntityRepository<Chart> {
   private static final Fields CHART_UPDATE_FIELDS = new Fields(ChartResource.FIELD_LIST, "owner");
   private static final Fields CHART_PATCH_FIELDS = new Fields(ChartResource.FIELD_LIST, "owner,service,tags");
@@ -66,9 +68,9 @@ public class ChartRepository extends EntityRepository<Chart> {
 
   @Override
   public void prepare(Chart chart) throws IOException {
-    chart.setService(getService(chart.getService()));
+    populateService(chart);
     chart.setFullyQualifiedName(getFQN(chart));
-    EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), chart.getOwner()); // Validate owner
+    EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), chart.getOwner()); // Validate and populate owner
     chart.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), chart.getTags()));
   }
 
@@ -149,18 +151,24 @@ public class ChartRepository extends EntityRepository<Chart> {
 
   private EntityReference getService(Chart chart) throws IOException {
     EntityReference ref = EntityUtil.getService(dao.relationshipDAO(), chart.getId(), Entity.DASHBOARD_SERVICE);
-    return getService(Objects.requireNonNull(ref));
+    DashboardService service = getService(ref.getId(), ref.getType());
+    ref.setName(service.getName());
+    ref.setDescription(service.getDescription());
+    return ref;
   }
 
-  private EntityReference getService(EntityReference service) throws IOException {
+  private void populateService(Chart chart) throws IOException {
-    if (service.getType().equalsIgnoreCase(Entity.DASHBOARD_SERVICE)) {
+    DashboardService service = getService(chart.getService().getId(), chart.getService().getType());
-      DashboardService serviceInstance = dao.dashboardServiceDAO().findEntityById(service.getId());
+    chart.setService(new DashboardServiceEntityInterface(service).getEntityReference());
-      service.setDescription(serviceInstance.getDescription());
+    chart.setServiceType(service.getServiceType());
-      service.setName(serviceInstance.getName());
+  }
-    } else {
+
-      throw new IllegalArgumentException(String.format("Invalid service type %s for the chart", service.getType()));
+  private DashboardService getService(UUID serviceId, String serviceType) throws IOException {
+    if (serviceType.equalsIgnoreCase(Entity.DASHBOARD_SERVICE)) {
+      return dao.dashboardServiceDAO().findEntityById(serviceId);
     }
-    return service;
+    throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(serviceType,
+            Entity.DASHBOARD_SERVICE));
   }
 
   public static class ChartEntityInterface implements EntityInterface<Chart> {

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java

@@ -17,7 +17,10 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.entity.data.Dashboard;
+import org.openmetadata.catalog.entity.services.DashboardService;
+import org.openmetadata.catalog.exception.CatalogExceptionMessage;
 import org.openmetadata.catalog.exception.EntityNotFoundException;
+import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface;
 import org.openmetadata.catalog.resources.dashboards.DashboardResource;
 import org.openmetadata.catalog.type.ChangeDescription;
 import org.openmetadata.catalog.type.EntityReference;
@@ -34,7 +37,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.UUID;
 
@@ -105,15 +107,23 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
 
   private EntityReference getService(Dashboard dashboard) throws IOException {
     EntityReference ref = EntityUtil.getService(dao.relationshipDAO(), dashboard.getId(), Entity.DASHBOARD_SERVICE);
-    return getService(Objects.requireNonNull(ref));
+    DashboardService service = getService(ref.getId(), ref.getType());
+    ref.setName(service.getName());
+    ref.setDescription(service.getDescription());
+    return ref;
   }
 
-  private EntityReference getService(EntityReference service) throws IOException {
+  private void populateService(Dashboard dashboard) throws IOException {
-    if (service.getType().equalsIgnoreCase(Entity.DASHBOARD_SERVICE)) {
+    DashboardService service = getService(dashboard.getService().getId(), dashboard.getService().getType());
-      return dao.dashboardServiceDAO().findEntityReferenceById(service.getId());
+    dashboard.setService(new DashboardServiceEntityInterface(service).getEntityReference());
-    } else {
+    dashboard.setServiceType(service.getServiceType());
-      throw new IllegalArgumentException(String.format("Invalid service type %s for the dashboard", service.getType()));
+  }
+
+  private DashboardService getService(UUID serviceId, String entityType) throws IOException {
+    if (entityType.equalsIgnoreCase(Entity.DASHBOARD_SERVICE)) {
+      return dao.dashboardServiceDAO().findEntityById(serviceId);
     }
+    throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(entityType, Entity.DASHBOARD));
   }
 
   public void setService(Dashboard dashboard, EntityReference service) throws IOException {
@@ -127,7 +137,7 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
 
   @Override
   public void prepare(Dashboard dashboard) throws IOException {
-    dashboard.setService(getService(dashboard.getService()));
+    populateService(dashboard);
     dashboard.setFullyQualifiedName(getFQN(dashboard));
     EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), dashboard.getOwner()); // Validate owner
     dashboard.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), dashboard.getTags()));
@@ -138,9 +148,10 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
     // Relationships and fields such as href are derived and not stored as part of json
     EntityReference owner = dashboard.getOwner();
     List<TagLabel> tags = dashboard.getTags();
+    EntityReference service = dashboard.getService();
 
     // Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships
-    dashboard.withOwner(null).withHref(null).withTags(null);
+    dashboard.withOwner(null).withHref(null).withTags(null).withService(null);
 
     if (update) {
       dao.dashboardDAO().update(dashboard.getId(), JsonUtils.pojoToJson(dashboard));
@@ -149,7 +160,7 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
     }
 
     // Restore the relationships
-    dashboard.withOwner(owner).withTags(tags);
+    dashboard.withOwner(owner).withTags(tags).withService(service);
   }
 
   @Override

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseRepository.java

@@ -16,7 +16,10 @@ package org.openmetadata.catalog.jdbi3;
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.entity.data.Database;
+import org.openmetadata.catalog.entity.services.DatabaseService;
+import org.openmetadata.catalog.exception.CatalogExceptionMessage;
 import org.openmetadata.catalog.exception.EntityNotFoundException;
+import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
 import org.openmetadata.catalog.resources.databases.DatabaseResource;
 import org.openmetadata.catalog.type.ChangeDescription;
 import org.openmetadata.catalog.type.EntityReference;
@@ -33,7 +36,6 @@ import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
-import java.util.Objects;
 import java.util.UUID;
 
 import static javax.ws.rs.core.Response.Status.CREATED;
@@ -73,7 +75,7 @@ public class DatabaseRepository extends EntityRepository<Database> {
 
   @Override
   public void prepare(Database database) throws IOException {
-    database.setService(getService(database.getService()));
+    populateService(database);
     database.setFullyQualifiedName(getFQN(database));
     database.setOwner(EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), database.getOwner())); // Validate owner
   }
@@ -160,15 +162,23 @@ public class DatabaseRepository extends EntityRepository<Database> {
 
   private EntityReference getService(Database database) throws IOException {
     EntityReference ref =  EntityUtil.getService(dao.relationshipDAO(), database.getId(), Entity.DATABASE_SERVICE);
-    return getService(Objects.requireNonNull(ref));
+    DatabaseService service = getService(ref.getId(), ref.getType());
+    ref.setName(service.getName());
+    ref.setDescription(service.getDescription());
+    return ref;
   }
 
-  private EntityReference getService(EntityReference service) throws IOException {
+  private void populateService(Database database) throws IOException {
-    if (service.getType().equalsIgnoreCase(Entity.DATABASE_SERVICE)) {
+    DatabaseService service = getService(database.getService().getId(), database.getService().getType());
-      return dao.dbServiceDAO().findEntityReferenceById(service.getId());
+    database.setService(new DatabaseServiceEntityInterface(service).getEntityReference());
-    } else {
+    database.setServiceType(service.getServiceType());
-      throw new IllegalArgumentException(String.format("Invalid service type %s for the database", service.getType()));
+  }
+
+  private DatabaseService getService(UUID serviceId, String entityType) throws IOException {
+    if (entityType.equalsIgnoreCase(Entity.DATABASE_SERVICE)) {
+      return dao.dbServiceDAO().findEntityById(serviceId);
     }
+    throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(entityType, Entity.DATABASE));
   }
 
   @Transaction

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java

@@ -15,6 +15,7 @@ package org.openmetadata.catalog.jdbi3;
 
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.catalog.Entity;
+import org.openmetadata.catalog.exception.CatalogExceptionMessage;
 import org.openmetadata.catalog.exception.EntityNotFoundException;
 import org.openmetadata.catalog.operations.workflows.Ingestion;
 import org.openmetadata.catalog.resources.operations.IngestionResource;
@@ -165,9 +166,9 @@ public class IngestionRepository extends EntityRepository<Ingestion> {
       return dao.dbServiceDAO().findEntityReferenceById(service.getId());
     } else if (service.getType().equalsIgnoreCase(Entity.DASHBOARD_SERVICE)) {
       return dao.dashboardServiceDAO().findEntityReferenceById(service.getId());
-    } else {
-      throw new IllegalArgumentException(String.format("Invalid service type %s for the ingestion", service.getType()));
     }
+    throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(service.getType(),
+            Entity.INGESTION));
   }
 
   public static class IngestionEntityInterface implements EntityInterface<Ingestion> {

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java

@@ -17,6 +17,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.entity.data.Location;
+import org.openmetadata.catalog.exception.CatalogExceptionMessage;
 import org.openmetadata.catalog.resources.locations.LocationResource;
 import org.openmetadata.catalog.type.ChangeDescription;
 import org.openmetadata.catalog.type.EntityReference;
@@ -207,10 +208,9 @@ public class LocationRepository extends EntityRepository<Location> {
   private EntityReference getService(EntityReference service) throws IOException {
     if (service.getType().equalsIgnoreCase(Entity.STORAGE_SERVICE)) {
       return dao.storageServiceDAO().findEntityReferenceById(service.getId());
-    } else {
-      throw new IllegalArgumentException(String.format("Invalid service type %s for the location",
-              service.getType()));
     }
+    throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(service.getType(),
+            Entity.LOCATION));
   }
 
   public void setService(Location location, EntityReference service) throws IOException {

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MetricsRepository.java

@@ -15,6 +15,7 @@ package org.openmetadata.catalog.jdbi3;
 
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.entity.data.Metrics;
+import org.openmetadata.catalog.exception.CatalogExceptionMessage;
 import org.openmetadata.catalog.resources.metrics.MetricsResource;
 import org.openmetadata.catalog.type.ChangeDescription;
 import org.openmetadata.catalog.type.EntityReference;
@@ -108,9 +109,9 @@ public class MetricsRepository extends EntityRepository<Metrics> {
   private EntityReference getService(EntityReference service) throws IOException { // Get service by service Id
     if (service.getType().equalsIgnoreCase(Entity.DASHBOARD_SERVICE)) {
       return dao.dbServiceDAO().findEntityReferenceById(service.getId());
-    } else {
-      throw new IllegalArgumentException(String.format("Invalid service type %s for the database", service.getType()));
     }
+    throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(service.getType(),
+            Entity.METRICS));
   }
 
   private EntityReference getOwner(Metrics metrics) throws IOException {

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java

@@ -18,7 +18,9 @@ import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.entity.data.Pipeline;
 import org.openmetadata.catalog.entity.services.PipelineService;
+import org.openmetadata.catalog.exception.CatalogExceptionMessage;
 import org.openmetadata.catalog.exception.EntityNotFoundException;
+import org.openmetadata.catalog.jdbi3.PipelineServiceRepository.PipelineServiceEntityInterface;
 import org.openmetadata.catalog.resources.pipelines.PipelineResource;
 import org.openmetadata.catalog.type.ChangeDescription;
 import org.openmetadata.catalog.type.EntityReference;
@@ -36,7 +38,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.UUID;
 import java.util.stream.Collectors;
@@ -112,10 +113,9 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
 
   @Override
   public void prepare(Pipeline pipeline) throws IOException {
-    pipeline.setService(getService(pipeline.getService()));
+    populateService(pipeline);
     pipeline.setFullyQualifiedName(getFQN(pipeline));
     EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), pipeline.getOwner()); // Validate owner
-    getService(pipeline.getService());
     pipeline.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), pipeline.getTags()));
   }
 
@@ -160,18 +160,23 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
   private EntityReference getService(Pipeline pipeline) throws IOException {
     EntityReference ref = EntityUtil.getService(dao.relationshipDAO(), pipeline.getId(),
             Entity.PIPELINE_SERVICE);
-    return getService(Objects.requireNonNull(ref));
+    PipelineService service = getService(ref.getId(), ref.getType());
+    ref.setName(service.getName());
+    ref.setDescription(service.getDescription());
+    return ref;
   }
 
-  private EntityReference getService(EntityReference service) throws IOException {
+  private void populateService(Pipeline pipeline) throws IOException {
-    if (service.getType().equalsIgnoreCase(Entity.PIPELINE_SERVICE)) {
+    PipelineService service = getService(pipeline.getService().getId(), pipeline.getService().getType());
-      PipelineService serviceInstance = dao.pipelineServiceDAO().findEntityById(service.getId());
+    pipeline.setService(new PipelineServiceEntityInterface(service).getEntityReference());
-      service.setDescription(serviceInstance.getDescription());
+    pipeline.setServiceType(service.getServiceType());
-      service.setName(serviceInstance.getName());
+  }
-    } else {
+
-      throw new IllegalArgumentException(String.format("Invalid service type %s for the pipeline", service.getType()));
+  private PipelineService getService(UUID serviceId, String entityType) throws IOException {
+    if (entityType.equalsIgnoreCase(Entity.PIPELINE_SERVICE)) {
+      return dao.pipelineServiceDAO().findEntityById(serviceId);
     }
-    return service;
+    throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(entityType, Entity.PIPELINE));
   }
 
   private EntityReference getOwner(Pipeline pipeline) throws IOException {

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java

@@ -18,8 +18,10 @@ import org.apache.commons.codec.binary.Hex;
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.entity.data.Table;
+import org.openmetadata.catalog.entity.services.DatabaseService;
 import org.openmetadata.catalog.exception.CatalogExceptionMessage;
 import org.openmetadata.catalog.exception.EntityNotFoundException;
+import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
 import org.openmetadata.catalog.resources.databases.TableResource;
 import org.openmetadata.catalog.type.ChangeDescription;
 import org.openmetadata.catalog.type.Column;
@@ -86,12 +88,13 @@ public class TableRepository extends EntityRepository<Table> {
   @Override
   public Table setFields(Table table, Fields fields) throws IOException, ParseException {
     table.setColumns(table.getColumns());
+    table.setDatabase(getDatabase(table.getId()));
+    table.setService(getService(table));
     table.setTableConstraints(fields.contains("tableConstraints") ? table.getTableConstraints() : null);
     table.setOwner(fields.contains("owner") ? getOwner(table) : null);
     table.setFollowers(fields.contains("followers") ? getFollowers(table) : null);
     table.setUsageSummary(fields.contains("usageSummary") ? EntityUtil.getLatestUsage(dao.usageDAO(), table.getId()) :
             null);
-    table.setDatabase(fields.contains("database") ? getDatabase(table.getId()) : null);
     table.setTags(fields.contains("tags") ? getTags(table.getFullyQualifiedName()) : null);
     getColumnTags(fields.contains("tags"), table.getColumns());
     table.setJoins(fields.contains("joins") ? getJoins(table) : null);
@@ -259,6 +262,7 @@ public class TableRepository extends EntityRepository<Table> {
   @Override
   public void prepare(Table table) throws IOException {
     table.setDatabase(dao.databaseDAO().findEntityReferenceById(table.getDatabase().getId()));
+    populateService(table);
 
     // Set data in table entity based on database relationship
     table.setFullyQualifiedName(getFQN(table));
@@ -274,15 +278,41 @@ public class TableRepository extends EntityRepository<Table> {
     addDerivedTags(table.getColumns());
   }
 
+  private DatabaseService getService(UUID serviceId, String entityType) throws IOException {
+    if (entityType.equalsIgnoreCase(Entity.DATABASE_SERVICE)) {
+      return dao.dbServiceDAO().findEntityById(serviceId);
+    }
+    throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(entityType, Entity.TABLE));
+  }
+
+  private EntityReference getService(Table table) throws IOException {
+    EntityReference ref = EntityUtil.getService(dao.relationshipDAO(), table.getDatabase().getId(),
+            Entity.DATABASE_SERVICE);
+    DatabaseService service = getService(ref.getId(), ref.getType());
+    ref.setName(service.getName());
+    ref.setDescription(service.getDescription());
+    return ref;
+  }
+
+  private void populateService(Table table) throws IOException {
+    // Find database service from the database that table is contained in
+    String serviceId = dao.relationshipDAO().findFrom(table.getDatabase().getId().toString(),
+            Relationship.CONTAINS.ordinal(), Entity.DATABASE_SERVICE).get(0);
+    DatabaseService service = dao.dbServiceDAO().findEntityById(UUID.fromString(serviceId));
+    table.setService(new DatabaseServiceEntityInterface(service).getEntityReference());
+    table.setServiceType(service.getServiceType());
+  }
+
   @Override
   public void storeEntity(Table table, boolean update) throws IOException {
     // Relationships and fields such as href are derived and not stored as part of json
     EntityReference owner = table.getOwner();
     EntityReference database = table.getDatabase();
     List<TagLabel> tags = table.getTags();
+    EntityReference service = table.getService();
 
     // Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships
-    table.withOwner(null).withDatabase(null).withHref(null).withTags(null);
+    table.withOwner(null).withDatabase(null).withHref(null).withTags(null).withService(null);
 
     // Don't store column tags as JSON but build it on the fly based on relationships
     List<Column> columnWithTags = table.getColumns();
@@ -296,8 +326,7 @@ public class TableRepository extends EntityRepository<Table> {
     }
 
     // Restore the relationships
-    table.withOwner(owner).withDatabase(database).withTags(tags);
+    table.withOwner(owner).withDatabase(database).withTags(tags).withColumns(columnWithTags).withService(service);
-    table.setColumns(columnWithTags);
   }
 
   @Override

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java

@@ -17,7 +17,9 @@ import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.entity.data.Topic;
 import org.openmetadata.catalog.entity.services.MessagingService;
+import org.openmetadata.catalog.exception.CatalogExceptionMessage;
 import org.openmetadata.catalog.exception.EntityNotFoundException;
+import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface;
 import org.openmetadata.catalog.resources.topics.TopicResource;
 import org.openmetadata.catalog.type.ChangeDescription;
 import org.openmetadata.catalog.type.EntityReference;
@@ -32,7 +34,6 @@ import java.net.URI;
 import java.text.ParseException;
 import java.util.Date;
 import java.util.List;
-import java.util.Objects;
 import java.util.UUID;
 
 import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound;
@@ -70,8 +71,9 @@ public class TopicRepository extends EntityRepository<Topic> {
 
   @Override
   public void prepare(Topic topic) throws IOException {
-    EntityReference messagingService = getService(topic.getService());
+    MessagingService messagingService = getService(topic.getService().getId(), topic.getService().getType());
-    topic.setService(messagingService);
+    topic.setService(new MessagingServiceEntityInterface(messagingService).getEntityReference());
+    topic.setServiceType(messagingService.getServiceType());
     topic.setFullyQualifiedName(getFQN(topic));
     EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), topic.getOwner()); // Validate owner
     topic.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), topic.getTags()));
@@ -117,7 +119,6 @@ public class TopicRepository extends EntityRepository<Topic> {
 
   private void setOwner(Topic topic, EntityReference owner) {
     EntityUtil.setOwner(dao.relationshipDAO(), topic.getId(), Entity.TOPIC, owner);
-    topic.setOwner(owner);
   }
 
   @Override
@@ -148,24 +149,23 @@ public class TopicRepository extends EntityRepository<Topic> {
   }
 
   private EntityReference getService(Topic topic) throws IOException {
-    return topic == null ? null : getService(Objects.requireNonNull(EntityUtil.getService(dao.relationshipDAO(),
+    if (topic == null) {
-            topic.getId())));
+      return null;
+    }
+    // Find service by topic Id
+    EntityReference service = EntityUtil.getService(dao.relationshipDAO(), topic.getId());
+    return new MessagingServiceEntityInterface(getService(service.getId(), service.getType())).getEntityReference();
   }
 
-  private EntityReference getService(EntityReference service) throws IOException {
+  private MessagingService getService(UUID serviceId, String entityType) throws IOException {
-    if (service.getType().equalsIgnoreCase(Entity.MESSAGING_SERVICE)) {
+    if (entityType.equalsIgnoreCase(Entity.MESSAGING_SERVICE)) {
-      MessagingService serviceInstance = dao.messagingServiceDAO().findEntityById(service.getId());
+      return dao.messagingServiceDAO().findEntityById(serviceId);
-      service.setDescription(serviceInstance.getDescription());
-      service.setName(serviceInstance.getName());
-    } else {
-      throw new IllegalArgumentException(String.format("Invalid service type %s for the topic", service.getType()));
     }
-    return service;
+    throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(entityType, Entity.TOPIC));
   }
 
   public void setService(Topic topic, EntityReference service) throws IOException {
     if (service != null && topic != null) {
-      getService(service); // Populate service details
       dao.relationshipDAO().insert(service.getId().toString(), topic.getId().toString(), service.getType(),
               Entity.TOPIC, Relationship.CONTAINS.ordinal());
       topic.setService(service);

catalog-rest-service/src/main/resources/json/schema/entity/data/chart.json

@@ -128,6 +128,10 @@
       "description": "Link to service where this dashboard is hosted in.",
       "$ref" : "../../type/entityReference.json"
     },
+    "serviceType" : {
+      "description": "Service type where this chart is hosted in.",
+      "$ref" : "../services/dashboardService.json#/definitions/dashboardServiceType"
+    },
     "usageSummary" : {
       "description": "Latest usage information for this database.",
       "$ref": "../../type/usageDetails.json",

catalog-rest-service/src/main/resources/json/schema/entity/data/dashboard.json

@@ -79,6 +79,10 @@
       "description": "Link to service where this dashboard is hosted in.",
       "$ref" : "../../type/entityReference.json"
     },
+    "serviceType" : {
+      "description": "Service type where this dashboard is hosted in.",
+      "$ref" : "../services/dashboardService.json#/definitions/dashboardServiceType"
+    },
     "usageSummary" : {
       "description": "Latest usage information for this database.",
       "$ref": "../../type/usageDetails.json",

catalog-rest-service/src/main/resources/json/schema/entity/data/database.json

@@ -59,6 +59,10 @@
       "description": "Link to the database cluster/service where this database is hosted in.",
       "$ref" : "../../type/entityReference.json"
     },
+    "serviceType" : {
+      "description": "Service type where this database is hosted in.",
+      "$ref" : "../services/databaseService.json#/definitions/databaseServiceType"
+    },
     "location": {
       "description": "Reference to the Location that contains this database.",
       "$ref": "../../type/entityReference.json"

catalog-rest-service/src/main/resources/json/schema/entity/data/location.json

@@ -93,6 +93,10 @@
       "description": "Link to the database cluster/service where this database is hosted in.",
       "$ref" : "../../type/entityReference.json"
     },
+    "serviceType" : {
+      "description": "Service type where this storage location is hosted in.",
+      "$ref" : "../../type/storage.json#/definitions/storageServiceType"
+    },
     "changeDescription": {
       "description" : "Change that lead to this version of the entity.",
       "$ref": "../../type/entityHistory.json#/definitions/changeDescription"

catalog-rest-service/src/main/resources/json/schema/entity/data/pipeline.json

@@ -145,6 +145,10 @@
       "description": "Link to service where this pipeline is hosted in.",
       "$ref" : "../../type/entityReference.json"
     },
+    "serviceType" : {
+      "description": "Service type where this pipeline is hosted in.",
+      "$ref" : "../services/pipelineService.json#/definitions/pipelineServiceType"
+    },
     "changeDescription": {
       "description" : "Change that lead to this version of the entity.",
       "$ref": "../../type/entityHistory.json#/definitions/changeDescription"

catalog-rest-service/src/main/resources/json/schema/entity/data/table.json

@@ -442,6 +442,14 @@
       "description": "Reference to Database that contains this table.",
       "$ref": "../../type/entityReference.json"
     },
+    "service": {
+      "description": "Link to Database service this table is hosted in.",
+      "$ref": "../../type/entityReference.json"
+    },
+    "serviceType" : {
+      "description": "Service type this table is hosted in.",
+      "$ref" : "../services/databaseService.json#/definitions/databaseServiceType"
+    },
     "location": {
       "description": "Reference to the Location that contains this table.",
       "$ref": "../../type/entityReference.json"

catalog-rest-service/src/main/resources/json/schema/entity/data/topic.json

@@ -83,6 +83,10 @@
       "description": "Link to the messaging cluster/service where this topic is hosted in.",
       "$ref": "../../type/entityReference.json"
     },
+    "serviceType" : {
+      "description": "Service type where this topic is hosted in.",
+      "$ref" : "../services/messagingService.json#/definitions/messagingServiceType"
+    },
     "partitions" : {
       "description" : "Number of partitions into which the topic is divided.",
       "type" : "integer",

catalog-rest-service/src/main/resources/json/schema/type/entityReference.json

@@ -20,7 +20,7 @@
       "$ref": "basic.json#/definitions/uuid"
     },
     "type": {
-      "description": "Entity type/class name - Examples: `database`, `table`, `metrics`, `redshift`, `mysql`, `bigquery`, `snowflake`...",
+      "description": "Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, `dashboardService`...",
       "type": "string"
     },
     "name": {

catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/charts/ChartResourceTest.java

@@ -209,20 +209,7 @@ public class ChartResourceTest extends EntityResourceTest<Chart> {
             getChart(chart.getId(), fields, adminAuthHeaders());
     assertNotNull(chart.getOwner());
     assertNotNull(chart.getService()); // We always return the service
-
+    assertNotNull(chart.getServiceType()); // We always return the service
-    // .../charts?fields=owner,service
-    fields = "owner,service";
-    chart = byName ? getChartByName(chart.getFullyQualifiedName(), fields, adminAuthHeaders()) :
-            getChart(chart.getId(), fields, adminAuthHeaders());
-    assertNotNull(chart.getOwner());
-    assertNotNull(chart.getService());
-
-    // .../charts?fields=owner,service
-    fields = "owner,service";
-    chart = byName ? getChartByName(chart.getFullyQualifiedName(), fields, adminAuthHeaders()) :
-            getChart(chart.getId(), fields, adminAuthHeaders());
-    assertNotNull(chart.getOwner());
-    assertNotNull(chart.getService());
   }
 
   public static void getChart(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
@@ -272,6 +259,7 @@ public class ChartResourceTest extends EntityResourceTest<Chart> {
     CreateChart createRequest = (CreateChart) request;
     validateCommonEntityFields(getEntityInterface(chart), createRequest.getDescription(),
             TestUtils.getPrincipal(authHeaders), createRequest.getOwner());
+    assertNotNull(chart.getServiceType());
     assertService(createRequest.getService(), chart.getService());
   }
 

catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java

@@ -63,6 +63,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.openmetadata.catalog.exception.CatalogExceptionMessage.ENTITY_ALREADY_EXISTS;
 import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound;
+import static org.openmetadata.catalog.exception.CatalogExceptionMessage.invalidServiceEntity;
 import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
 import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
 import static org.openmetadata.catalog.util.TestUtils.assertResponse;
@@ -156,9 +157,8 @@ public class DashboardResourceTest extends EntityResourceTest<Dashboard> {
     CreateDashboard create = create(test).withService(SUPERSET_INVALID_SERVICE_REFERENCE);
     HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
             createDashboard(create, adminAuthHeaders()));
-    TestUtils.assertResponseContains(exception, BAD_REQUEST, String.format("Invalid service type %s",
+    TestUtils.assertResponseContains(exception, BAD_REQUEST,
-            SUPERSET_INVALID_SERVICE_REFERENCE.getType()));
+            invalidServiceEntity(SUPERSET_INVALID_SERVICE_REFERENCE.getType(), Entity.DASHBOARD));
-
   }
 
   @Test
@@ -283,22 +283,17 @@ public class DashboardResourceTest extends EntityResourceTest<Dashboard> {
             getDashboard(dashboard.getId(), fields, adminAuthHeaders());
     assertNotNull(dashboard.getOwner());
     assertNotNull(dashboard.getService()); // We always return the service
+    assertNotNull(dashboard.getServiceType());
     assertNull(dashboard.getCharts());
-
+    assertNull(dashboard.getUsageSummary());
-    // .../Dashboards?fields=owner,service
-    fields = "owner,service";
-    dashboard = byName ? getDashboardByName(dashboard.getFullyQualifiedName(), fields, adminAuthHeaders()) :
-            getDashboard(dashboard.getId(), fields, adminAuthHeaders());
-    assertNotNull(dashboard.getOwner());
-    assertNotNull(dashboard.getService());
-    assertNull(dashboard.getCharts());
 
     // .../Dashboards?fields=owner,service,tables
     fields = "owner,service,charts,usageSummary";
     dashboard = byName ? getDashboardByName(dashboard.getFullyQualifiedName(), fields, adminAuthHeaders()) :
             getDashboard(dashboard.getId(), fields, adminAuthHeaders());
     assertNotNull(dashboard.getOwner());
-    assertNotNull(dashboard.getService());
+    assertNotNull(dashboard.getService());  // We always return the service
+    assertNotNull(dashboard.getServiceType());
     assertNotNull(dashboard.getCharts());
     TestUtils.validateEntityReference(dashboard.getCharts());
     assertNotNull(dashboard.getUsageSummary());
@@ -361,6 +356,7 @@ public class DashboardResourceTest extends EntityResourceTest<Dashboard> {
     CreateDashboard createRequest = (CreateDashboard) request;
     validateCommonEntityFields(getEntityInterface(dashboard), createRequest.getDescription(),
             TestUtils.getPrincipal(authHeaders), createRequest.getOwner());
+    assertNotNull(dashboard.getServiceType());
     assertService(createRequest.getService(), dashboard.getService());
     validateDashboardCharts(dashboard, createRequest.getCharts());
     TestUtils.validateTags(createRequest.getTags(), dashboard.getTags());

catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/DatabaseResourceTest.java

@@ -208,22 +208,17 @@ public class DatabaseResourceTest extends EntityResourceTest<Database> {
             getDatabase(database.getId(), fields, adminAuthHeaders());
     assertNotNull(database.getOwner());
     assertNotNull(database.getService()); // We always return the service
+    assertNotNull(database.getServiceType());
     assertNull(database.getTables());
-
+    assertNull(database.getUsageSummary());
-    // .../databases?fields=owner,service
-    fields = "owner,service";
-    database = byName ? getDatabaseByName(database.getFullyQualifiedName(), fields, adminAuthHeaders()) :
-            getDatabase(database.getId(), fields, adminAuthHeaders());
-    assertNotNull(database.getOwner());
-    assertNotNull(database.getService());
-    assertNull(database.getTables());
 
     // .../databases?fields=owner,service,tables
     fields = "owner,service,tables,usageSummary";
     database = byName ? getDatabaseByName(database.getFullyQualifiedName(), fields, adminAuthHeaders()) :
             getDatabase(database.getId(), fields, adminAuthHeaders());
     assertNotNull(database.getOwner());
-    assertNotNull(database.getService());
+    assertNotNull(database.getService()); // We always return the service
+    assertNotNull(database.getServiceType());
     assertNotNull(database.getTables());
     TestUtils.validateEntityReference(database.getTables());
     assertNotNull(database.getUsageSummary());
@@ -270,13 +265,14 @@ public class DatabaseResourceTest extends EntityResourceTest<Database> {
   }
 
   @Override
-  public void validateCreatedEntity(Database createdEntity, Object request, Map<String, String> authHeaders) {
+  public void validateCreatedEntity(Database database, Object request, Map<String, String> authHeaders) {
     CreateDatabase createRequest = (CreateDatabase) request;
-    validateCommonEntityFields(getEntityInterface(createdEntity), createRequest.getDescription(),
+    validateCommonEntityFields(getEntityInterface(database), createRequest.getDescription(),
             TestUtils.getPrincipal(authHeaders), createRequest.getOwner());
 
     // Validate service
-    assertService(createRequest.getService(), createdEntity.getService());
+    assertNotNull(database.getServiceType());
+    assertService(createRequest.getService(), database.getService());
   }
 
   @Override

catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java

@@ -1084,16 +1084,15 @@ public class TableResourceTest extends EntityResourceTest<Table> {
     } else {
       assertNull(table.getTableConstraints());
     }
-    if (fields.contains("database")) {
-      assertNotNull(table.getDatabase());
-    } else {
-      assertNull(table.getDatabase());
-    }
     if (fields.contains("tags")) {
       assertNotNull(table.getTags());
     } else {
       assertNull(table.getTags());
     }
+    // Default fields that are always returned
+    assertNotNull(table.getDatabase());
+    assertNotNull(table.getService());
+    assertNotNull(table.getServiceType());
   }
 
   /** Validate returned fields GET .../tables/{id}?fields="..." or GET .../tables/name/{fqn}?fields="..." */
@@ -1273,6 +1272,8 @@ public class TableResourceTest extends EntityResourceTest<Table> {
     assertEquals(createRequest.getTableConstraints(), createdEntity.getTableConstraints());
     TestUtils.validateTags(createRequest.getTags(), createdEntity.getTags());
     TestUtils.validateEntityReference(createdEntity.getFollowers());
+    assertNotNull(createdEntity.getService());
+    assertNotNull(createdEntity.getServiceType());
   }
 
   @Override

catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/pipelines/PipelineResourceTest.java

@@ -93,6 +93,7 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline> {
     validateCommonEntityFields(getEntityInterface(pipeline), createRequest.getDescription(),
             TestUtils.getPrincipal(authHeaders), createRequest.getOwner());
     assertEquals(createRequest.getDisplayName(), pipeline.getDisplayName());
+    assertNotNull(pipeline.getServiceType());
     assertService(createRequest.getService(), pipeline.getService());
     assertEquals(createRequest.getTasks(), pipeline.getTasks());
     TestUtils.validateTags(createRequest.getTags(), pipeline.getTags());
@@ -339,14 +340,7 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline> {
             getPipeline(pipeline.getId(), fields, adminAuthHeaders());
     assertNotNull(pipeline.getOwner());
     assertNotNull(pipeline.getService()); // We always return the service
-    assertNull(pipeline.getTasks());
+    assertNotNull(pipeline.getServiceType());
-
-    // .../Pipelines?fields=owner,service
-    fields = "owner,service";
-    pipeline = byName ? getPipelineByName(pipeline.getFullyQualifiedName(), fields, adminAuthHeaders()) :
-            getPipeline(pipeline.getId(), fields, adminAuthHeaders());
-    assertNotNull(pipeline.getOwner());
-    assertNotNull(pipeline.getService());
     assertNull(pipeline.getTasks());
 
     // .../Pipelines?fields=owner,service,tables
@@ -354,7 +348,8 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline> {
     pipeline = byName ? getPipelineByName(pipeline.getFullyQualifiedName(), fields, adminAuthHeaders()) :
             getPipeline(pipeline.getId(), fields, adminAuthHeaders());
     assertNotNull(pipeline.getOwner());
-    assertNotNull(pipeline.getService());
+    assertNotNull(pipeline.getService()); // We always return the service
+    assertNotNull(pipeline.getServiceType());
     assertNotNull(pipeline.getTasks());
   }
 

catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/topics/TopicResourceTest.java

@@ -193,20 +193,7 @@ public class TopicResourceTest extends EntityResourceTest<Topic> {
             getTopic(topic.getId(), fields, adminAuthHeaders());
     assertNotNull(topic.getOwner());
     assertNotNull(topic.getService()); // We always return the service
-
+    assertNotNull(topic.getServiceType());
-    // .../topics?fields=owner,service
-    fields = "owner,service";
-    topic = byName ? getTopicByName(topic.getFullyQualifiedName(), fields, adminAuthHeaders()) :
-            getTopic(topic.getId(), fields, adminAuthHeaders());
-    assertNotNull(topic.getOwner());
-    assertNotNull(topic.getService());
-
-    // .../topics?fields=owner,service
-    fields = "owner,service";
-    topic = byName ? getTopicByName(topic.getFullyQualifiedName(), fields, adminAuthHeaders()) :
-            getTopic(topic.getId(), fields, adminAuthHeaders());
-    assertNotNull(topic.getOwner());
-    assertNotNull(topic.getService());
   }
 
   public static void getTopic(UUID id, Map<String, String> authHeaders) throws HttpResponseException {