yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-05 07:38:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #1508 Add serviceType and service reference for Chart, Dashboard, Database, Pipeline, Table, and Topic (#1510)

Browse Source 
Co-authored-by: sureshms <suresh@getcollate.io>
This commit is contained in:
Suresh Srinivas 2021-12-01 16:33:34 -08:00 committed by GitHub
parent 2171236ada
commit 4b94a3fda4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 202 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.idea/encodings.xml generated
View File

@ -5,5 +5,6 @@
<file url="file://$PROJECT_DIR$/catalog-rest-service/src/main/resources" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/catalog-rest-service/src/main/resources/json/data" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/common/src/main/java" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/openmetadata-ui/src/main/resources/ui/dist" charset="UTF-8" />
</component>
</project>

2
.idea/misc.xml generated
View File

@ -8,5 +8,5 @@
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="11" languageLevel="JDK_11" project-jdk-type="JavaSDK" />
<component name="ProjectRootManager" version="2" languageLevel="JDK_11" project-jdk-name="11" project-jdk-type="JavaSDK" />
</project>

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogExceptionMessage.java
View File

@ -56,4 +56,8 @@ public final class CatalogExceptionMessage {
public static String entityVersionNotFound(String entity, String id, Double version) {
return String.format("%s instance for %s and version %s not found", StringUtils.capitalize(entity), id, version);
}
public static String invalidServiceEntity(String serviceEntity, String entity) {
return String.format("Invalid service entity type %s for %s", serviceEntity, entity);
}
}

32
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java
View File

@ -18,7 +18,9 @@ import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Chart;
import org.openmetadata.catalog.entity.services.DashboardService;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface;
import org.openmetadata.catalog.resources.charts.ChartResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
@ -33,11 +35,11 @@ import java.net.URI;
import java.text.ParseException;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound;
public class ChartRepository extends EntityRepository<Chart> {
private static final Fields CHART_UPDATE_FIELDS = new Fields(ChartResource.FIELD_LIST, "owner");
private static final Fields CHART_PATCH_FIELDS = new Fields(ChartResource.FIELD_LIST, "owner,service,tags");
@ -66,9 +68,9 @@ public class ChartRepository extends EntityRepository<Chart> {
@Override
public void prepare(Chart chart) throws IOException {
chart.setService(getService(chart.getService()));
populateService(chart);
chart.setFullyQualifiedName(getFQN(chart));
EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), chart.getOwner()); // Validate owner
EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), chart.getOwner()); // Validate and populate owner
chart.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), chart.getTags()));
}
@ -149,18 +151,24 @@ public class ChartRepository extends EntityRepository<Chart> {
private EntityReference getService(Chart chart) throws IOException {
EntityReference ref = EntityUtil.getService(dao.relationshipDAO(), chart.getId(), Entity.DASHBOARD_SERVICE);
return getService(Objects.requireNonNull(ref));
DashboardService service = getService(ref.getId(), ref.getType());
ref.setName(service.getName());
ref.setDescription(service.getDescription());
return ref;
}
private EntityReference getService(EntityReference service) throws IOException {
if (service.getType().equalsIgnoreCase(Entity.DASHBOARD_SERVICE)) {
DashboardService serviceInstance = dao.dashboardServiceDAO().findEntityById(service.getId());
service.setDescription(serviceInstance.getDescription());
service.setName(serviceInstance.getName());
} else {
throw new IllegalArgumentException(String.format("Invalid service type %s for the chart", service.getType()));
private void populateService(Chart chart) throws IOException {
DashboardService service = getService(chart.getService().getId(), chart.getService().getType());
chart.setService(new DashboardServiceEntityInterface(service).getEntityReference());
chart.setServiceType(service.getServiceType());
}
private DashboardService getService(UUID serviceId, String serviceType) throws IOException {
if (serviceType.equalsIgnoreCase(Entity.DASHBOARD_SERVICE)) {
return dao.dashboardServiceDAO().findEntityById(serviceId);
}
return service;
throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(serviceType,
Entity.DASHBOARD_SERVICE));
}
public static class ChartEntityInterface implements EntityInterface<Chart> {

31
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java
View File

@ -17,7 +17,10 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.entity.services.DashboardService;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface;
import org.openmetadata.catalog.resources.dashboards.DashboardResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
@ -34,7 +37,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
@ -105,15 +107,23 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
private EntityReference getService(Dashboard dashboard) throws IOException {
EntityReference ref = EntityUtil.getService(dao.relationshipDAO(), dashboard.getId(), Entity.DASHBOARD_SERVICE);
return getService(Objects.requireNonNull(ref));
DashboardService service = getService(ref.getId(), ref.getType());
ref.setName(service.getName());
ref.setDescription(service.getDescription());
return ref;
}
private EntityReference getService(EntityReference service) throws IOException {
if (service.getType().equalsIgnoreCase(Entity.DASHBOARD_SERVICE)) {
return dao.dashboardServiceDAO().findEntityReferenceById(service.getId());
} else {
throw new IllegalArgumentException(String.format("Invalid service type %s for the dashboard", service.getType()));
private void populateService(Dashboard dashboard) throws IOException {
DashboardService service = getService(dashboard.getService().getId(), dashboard.getService().getType());
dashboard.setService(new DashboardServiceEntityInterface(service).getEntityReference());
dashboard.setServiceType(service.getServiceType());
}
private DashboardService getService(UUID serviceId, String entityType) throws IOException {
if (entityType.equalsIgnoreCase(Entity.DASHBOARD_SERVICE)) {
return dao.dashboardServiceDAO().findEntityById(serviceId);
}
throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(entityType, Entity.DASHBOARD));
}
public void setService(Dashboard dashboard, EntityReference service) throws IOException {
@ -127,7 +137,7 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
@Override
public void prepare(Dashboard dashboard) throws IOException {
dashboard.setService(getService(dashboard.getService()));
populateService(dashboard);
dashboard.setFullyQualifiedName(getFQN(dashboard));
EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), dashboard.getOwner()); // Validate owner
dashboard.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), dashboard.getTags()));
@ -138,9 +148,10 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
// Relationships and fields such as href are derived and not stored as part of json
EntityReference owner = dashboard.getOwner();
List<TagLabel> tags = dashboard.getTags();
EntityReference service = dashboard.getService();
// Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships
dashboard.withOwner(null).withHref(null).withTags(null);
dashboard.withOwner(null).withHref(null).withTags(null).withService(null);
if (update) {
dao.dashboardDAO().update(dashboard.getId(), JsonUtils.pojoToJson(dashboard));
@ -149,7 +160,7 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
}
// Restore the relationships
dashboard.withOwner(owner).withTags(tags);
dashboard.withOwner(owner).withTags(tags).withService(service);
}
@Override

26
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseRepository.java
View File

@ -16,7 +16,10 @@ package org.openmetadata.catalog.jdbi3;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Database;
import org.openmetadata.catalog.entity.services.DatabaseService;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
import org.openmetadata.catalog.resources.databases.DatabaseResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
@ -33,7 +36,6 @@ import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import static javax.ws.rs.core.Response.Status.CREATED;
@ -73,7 +75,7 @@ public class DatabaseRepository extends EntityRepository<Database> {
@Override
public void prepare(Database database) throws IOException {
database.setService(getService(database.getService()));
populateService(database);
database.setFullyQualifiedName(getFQN(database));
database.setOwner(EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), database.getOwner())); // Validate owner
}
@ -160,15 +162,23 @@ public class DatabaseRepository extends EntityRepository<Database> {
private EntityReference getService(Database database) throws IOException {
EntityReference ref = EntityUtil.getService(dao.relationshipDAO(), database.getId(), Entity.DATABASE_SERVICE);
return getService(Objects.requireNonNull(ref));
DatabaseService service = getService(ref.getId(), ref.getType());
ref.setName(service.getName());
ref.setDescription(service.getDescription());
return ref;
}
private EntityReference getService(EntityReference service) throws IOException {
if (service.getType().equalsIgnoreCase(Entity.DATABASE_SERVICE)) {
return dao.dbServiceDAO().findEntityReferenceById(service.getId());
} else {
throw new IllegalArgumentException(String.format("Invalid service type %s for the database", service.getType()));
private void populateService(Database database) throws IOException {
DatabaseService service = getService(database.getService().getId(), database.getService().getType());
database.setService(new DatabaseServiceEntityInterface(service).getEntityReference());
database.setServiceType(service.getServiceType());
}
private DatabaseService getService(UUID serviceId, String entityType) throws IOException {
if (entityType.equalsIgnoreCase(Entity.DATABASE_SERVICE)) {
return dao.dbServiceDAO().findEntityById(serviceId);
}
throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(entityType, Entity.DATABASE));
}
@Transaction

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java
View File

@ -15,6 +15,7 @@ package org.openmetadata.catalog.jdbi3;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.operations.workflows.Ingestion;
import org.openmetadata.catalog.resources.operations.IngestionResource;
@ -165,9 +166,9 @@ public class IngestionRepository extends EntityRepository<Ingestion> {
return dao.dbServiceDAO().findEntityReferenceById(service.getId());
} else if (service.getType().equalsIgnoreCase(Entity.DASHBOARD_SERVICE)) {
return dao.dashboardServiceDAO().findEntityReferenceById(service.getId());
} else {
throw new IllegalArgumentException(String.format("Invalid service type %s for the ingestion", service.getType()));
}
throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(service.getType(),
Entity.INGESTION));
}
public static class IngestionEntityInterface implements EntityInterface<Ingestion> {

6
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java
View File

@ -17,6 +17,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Location;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.resources.locations.LocationResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
@ -207,10 +208,9 @@ public class LocationRepository extends EntityRepository<Location> {
private EntityReference getService(EntityReference service) throws IOException {
if (service.getType().equalsIgnoreCase(Entity.STORAGE_SERVICE)) {
return dao.storageServiceDAO().findEntityReferenceById(service.getId());
} else {
throw new IllegalArgumentException(String.format("Invalid service type %s for the location",
service.getType()));
}
throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(service.getType(),
Entity.LOCATION));
}
public void setService(Location location, EntityReference service) throws IOException {

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MetricsRepository.java
View File

@ -15,6 +15,7 @@ package org.openmetadata.catalog.jdbi3;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Metrics;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.resources.metrics.MetricsResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
@ -108,9 +109,9 @@ public class MetricsRepository extends EntityRepository<Metrics> {
private EntityReference getService(EntityReference service) throws IOException { // Get service by service Id
if (service.getType().equalsIgnoreCase(Entity.DASHBOARD_SERVICE)) {
return dao.dbServiceDAO().findEntityReferenceById(service.getId());
} else {
throw new IllegalArgumentException(String.format("Invalid service type %s for the database", service.getType()));
}
throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(service.getType(),
Entity.METRICS));
}
private EntityReference getOwner(Metrics metrics) throws IOException {

29
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java
View File

@ -18,7 +18,9 @@ import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Pipeline;
import org.openmetadata.catalog.entity.services.PipelineService;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.PipelineServiceRepository.PipelineServiceEntityInterface;
import org.openmetadata.catalog.resources.pipelines.PipelineResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
@ -36,7 +38,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
@ -112,10 +113,9 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
@Override
public void prepare(Pipeline pipeline) throws IOException {
pipeline.setService(getService(pipeline.getService()));
populateService(pipeline);
pipeline.setFullyQualifiedName(getFQN(pipeline));
EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), pipeline.getOwner()); // Validate owner
getService(pipeline.getService());
pipeline.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), pipeline.getTags()));
}
@ -160,18 +160,23 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
private EntityReference getService(Pipeline pipeline) throws IOException {
EntityReference ref = EntityUtil.getService(dao.relationshipDAO(), pipeline.getId(),
Entity.PIPELINE_SERVICE);
return getService(Objects.requireNonNull(ref));
PipelineService service = getService(ref.getId(), ref.getType());
ref.setName(service.getName());
ref.setDescription(service.getDescription());
return ref;
}
private EntityReference getService(EntityReference service) throws IOException {
if (service.getType().equalsIgnoreCase(Entity.PIPELINE_SERVICE)) {
PipelineService serviceInstance = dao.pipelineServiceDAO().findEntityById(service.getId());
service.setDescription(serviceInstance.getDescription());
service.setName(serviceInstance.getName());
} else {
throw new IllegalArgumentException(String.format("Invalid service type %s for the pipeline", service.getType()));
private void populateService(Pipeline pipeline) throws IOException {
PipelineService service = getService(pipeline.getService().getId(), pipeline.getService().getType());
pipeline.setService(new PipelineServiceEntityInterface(service).getEntityReference());
pipeline.setServiceType(service.getServiceType());
}
private PipelineService getService(UUID serviceId, String entityType) throws IOException {
if (entityType.equalsIgnoreCase(Entity.PIPELINE_SERVICE)) {
return dao.pipelineServiceDAO().findEntityById(serviceId);
}
return service;
throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(entityType, Entity.PIPELINE));
}
private EntityReference getOwner(Pipeline pipeline) throws IOException {

37
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java
View File

@ -18,8 +18,10 @@ import org.apache.commons.codec.binary.Hex;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.entity.services.DatabaseService;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
import org.openmetadata.catalog.resources.databases.TableResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.Column;
@ -86,12 +88,13 @@ public class TableRepository extends EntityRepository<Table> {
@Override
public Table setFields(Table table, Fields fields) throws IOException, ParseException {
table.setColumns(table.getColumns());
table.setDatabase(getDatabase(table.getId()));
table.setService(getService(table));
table.setTableConstraints(fields.contains("tableConstraints") ? table.getTableConstraints() : null);
table.setOwner(fields.contains("owner") ? getOwner(table) : null);
table.setFollowers(fields.contains("followers") ? getFollowers(table) : null);
table.setUsageSummary(fields.contains("usageSummary") ? EntityUtil.getLatestUsage(dao.usageDAO(), table.getId()) :
null);
table.setDatabase(fields.contains("database") ? getDatabase(table.getId()) : null);
table.setTags(fields.contains("tags") ? getTags(table.getFullyQualifiedName()) : null);
getColumnTags(fields.contains("tags"), table.getColumns());
table.setJoins(fields.contains("joins") ? getJoins(table) : null);
@ -259,6 +262,7 @@ public class TableRepository extends EntityRepository<Table> {
@Override
public void prepare(Table table) throws IOException {
table.setDatabase(dao.databaseDAO().findEntityReferenceById(table.getDatabase().getId()));
populateService(table);
// Set data in table entity based on database relationship
table.setFullyQualifiedName(getFQN(table));
@ -274,15 +278,41 @@ public class TableRepository extends EntityRepository<Table> {
addDerivedTags(table.getColumns());
}
private DatabaseService getService(UUID serviceId, String entityType) throws IOException {
if (entityType.equalsIgnoreCase(Entity.DATABASE_SERVICE)) {
return dao.dbServiceDAO().findEntityById(serviceId);
}
throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(entityType, Entity.TABLE));
}
private EntityReference getService(Table table) throws IOException {
EntityReference ref = EntityUtil.getService(dao.relationshipDAO(), table.getDatabase().getId(),
Entity.DATABASE_SERVICE);
DatabaseService service = getService(ref.getId(), ref.getType());
ref.setName(service.getName());
ref.setDescription(service.getDescription());
return ref;
}
private void populateService(Table table) throws IOException {
// Find database service from the database that table is contained in
String serviceId = dao.relationshipDAO().findFrom(table.getDatabase().getId().toString(),
Relationship.CONTAINS.ordinal(), Entity.DATABASE_SERVICE).get(0);
DatabaseService service = dao.dbServiceDAO().findEntityById(UUID.fromString(serviceId));
table.setService(new DatabaseServiceEntityInterface(service).getEntityReference());
table.setServiceType(service.getServiceType());
}
@Override
public void storeEntity(Table table, boolean update) throws IOException {
// Relationships and fields such as href are derived and not stored as part of json
EntityReference owner = table.getOwner();
EntityReference database = table.getDatabase();
List<TagLabel> tags = table.getTags();
EntityReference service = table.getService();
// Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships
table.withOwner(null).withDatabase(null).withHref(null).withTags(null);
table.withOwner(null).withDatabase(null).withHref(null).withTags(null).withService(null);
// Don't store column tags as JSON but build it on the fly based on relationships
List<Column> columnWithTags = table.getColumns();
@ -296,8 +326,7 @@ public class TableRepository extends EntityRepository<Table> {
}
// Restore the relationships
table.withOwner(owner).withDatabase(database).withTags(tags);
table.setColumns(columnWithTags);
table.withOwner(owner).withDatabase(database).withTags(tags).withColumns(columnWithTags).withService(service);
}
@Override

30
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java
View File

@ -17,7 +17,9 @@ import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Topic;
import org.openmetadata.catalog.entity.services.MessagingService;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface;
import org.openmetadata.catalog.resources.topics.TopicResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
@ -32,7 +34,6 @@ import java.net.URI;
import java.text.ParseException;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound;
@ -70,8 +71,9 @@ public class TopicRepository extends EntityRepository<Topic> {
@Override
public void prepare(Topic topic) throws IOException {
EntityReference messagingService = getService(topic.getService());
topic.setService(messagingService);
MessagingService messagingService = getService(topic.getService().getId(), topic.getService().getType());
topic.setService(new MessagingServiceEntityInterface(messagingService).getEntityReference());
topic.setServiceType(messagingService.getServiceType());
topic.setFullyQualifiedName(getFQN(topic));
EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), topic.getOwner()); // Validate owner
topic.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), topic.getTags()));
@ -117,7 +119,6 @@ public class TopicRepository extends EntityRepository<Topic> {
private void setOwner(Topic topic, EntityReference owner) {
EntityUtil.setOwner(dao.relationshipDAO(), topic.getId(), Entity.TOPIC, owner);
topic.setOwner(owner);
}
@Override
@ -148,24 +149,23 @@ public class TopicRepository extends EntityRepository<Topic> {
}
private EntityReference getService(Topic topic) throws IOException {
return topic == null ? null : getService(Objects.requireNonNull(EntityUtil.getService(dao.relationshipDAO(),
topic.getId())));
if (topic == null) {
return null;
}
// Find service by topic Id
EntityReference service = EntityUtil.getService(dao.relationshipDAO(), topic.getId());
return new MessagingServiceEntityInterface(getService(service.getId(), service.getType())).getEntityReference();
}
private EntityReference getService(EntityReference service) throws IOException {
if (service.getType().equalsIgnoreCase(Entity.MESSAGING_SERVICE)) {
MessagingService serviceInstance = dao.messagingServiceDAO().findEntityById(service.getId());
service.setDescription(serviceInstance.getDescription());
service.setName(serviceInstance.getName());
} else {
throw new IllegalArgumentException(String.format("Invalid service type %s for the topic", service.getType()));
private MessagingService getService(UUID serviceId, String entityType) throws IOException {
if (entityType.equalsIgnoreCase(Entity.MESSAGING_SERVICE)) {
return dao.messagingServiceDAO().findEntityById(serviceId);
}
return service;
throw new IllegalArgumentException(CatalogExceptionMessage.invalidServiceEntity(entityType, Entity.TOPIC));
}
public void setService(Topic topic, EntityReference service) throws IOException {
if (service != null && topic != null) {
getService(service); // Populate service details
dao.relationshipDAO().insert(service.getId().toString(), topic.getId().toString(), service.getType(),
Entity.TOPIC, Relationship.CONTAINS.ordinal());
topic.setService(service);

4
catalog-rest-service/src/main/resources/json/schema/entity/data/chart.json
View File

@ -128,6 +128,10 @@
"description": "Link to service where this dashboard is hosted in.",
"$ref" : "../../type/entityReference.json"
},
"serviceType" : {
"description": "Service type where this chart is hosted in.",
"$ref" : "../services/dashboardService.json#/definitions/dashboardServiceType"
},
"usageSummary" : {
"description": "Latest usage information for this database.",
"$ref": "../../type/usageDetails.json",

4
catalog-rest-service/src/main/resources/json/schema/entity/data/dashboard.json
View File

@ -79,6 +79,10 @@
"description": "Link to service where this dashboard is hosted in.",
"$ref" : "../../type/entityReference.json"
},
"serviceType" : {
"description": "Service type where this dashboard is hosted in.",
"$ref" : "../services/dashboardService.json#/definitions/dashboardServiceType"
},
"usageSummary" : {
"description": "Latest usage information for this database.",
"$ref": "../../type/usageDetails.json",

4
catalog-rest-service/src/main/resources/json/schema/entity/data/database.json
View File

@ -59,6 +59,10 @@
"description": "Link to the database cluster/service where this database is hosted in.",
"$ref" : "../../type/entityReference.json"
},
"serviceType" : {
"description": "Service type where this database is hosted in.",
"$ref" : "../services/databaseService.json#/definitions/databaseServiceType"
},
"location": {
"description": "Reference to the Location that contains this database.",
"$ref": "../../type/entityReference.json"

4
catalog-rest-service/src/main/resources/json/schema/entity/data/location.json
View File

@ -93,6 +93,10 @@
"description": "Link to the database cluster/service where this database is hosted in.",
"$ref" : "../../type/entityReference.json"
},
"serviceType" : {
"description": "Service type where this storage location is hosted in.",
"$ref" : "../../type/storage.json#/definitions/storageServiceType"
},
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"

4
catalog-rest-service/src/main/resources/json/schema/entity/data/pipeline.json
View File

@ -145,6 +145,10 @@
"description": "Link to service where this pipeline is hosted in.",
"$ref" : "../../type/entityReference.json"
},
"serviceType" : {
"description": "Service type where this pipeline is hosted in.",
"$ref" : "../services/pipelineService.json#/definitions/pipelineServiceType"
},
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"

8
catalog-rest-service/src/main/resources/json/schema/entity/data/table.json
View File

@ -442,6 +442,14 @@
"description": "Reference to Database that contains this table.",
"$ref": "../../type/entityReference.json"
},
"service": {
"description": "Link to Database service this table is hosted in.",
"$ref": "../../type/entityReference.json"
},
"serviceType" : {
"description": "Service type this table is hosted in.",
"$ref" : "../services/databaseService.json#/definitions/databaseServiceType"
},
"location": {
"description": "Reference to the Location that contains this table.",
"$ref": "../../type/entityReference.json"

4
catalog-rest-service/src/main/resources/json/schema/entity/data/topic.json
View File

@ -83,6 +83,10 @@
"description": "Link to the messaging cluster/service where this topic is hosted in.",
"$ref": "../../type/entityReference.json"
},
"serviceType" : {
"description": "Service type where this topic is hosted in.",
"$ref" : "../services/messagingService.json#/definitions/messagingServiceType"
},
"partitions" : {
"description" : "Number of partitions into which the topic is divided.",
"type" : "integer",

2
catalog-rest-service/src/main/resources/json/schema/type/entityReference.json
View File

@ -20,7 +20,7 @@
"$ref": "basic.json#/definitions/uuid"
},
"type": {
"description": "Entity type/class name - Examples: `database`, `table`, `metrics`, `redshift`, `mysql`, `bigquery`, `snowflake`...",
"description": "Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, `dashboardService`...",
"type": "string"
},
"name": {

16
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/charts/ChartResourceTest.java
View File

@ -209,20 +209,7 @@ public class ChartResourceTest extends EntityResourceTest<Chart> {
getChart(chart.getId(), fields, adminAuthHeaders());
assertNotNull(chart.getOwner());
assertNotNull(chart.getService()); // We always return the service
// .../charts?fields=owner,service
fields = "owner,service";
chart = byName ? getChartByName(chart.getFullyQualifiedName(), fields, adminAuthHeaders()) :
getChart(chart.getId(), fields, adminAuthHeaders());
assertNotNull(chart.getOwner());
assertNotNull(chart.getService());
// .../charts?fields=owner,service
fields = "owner,service";
chart = byName ? getChartByName(chart.getFullyQualifiedName(), fields, adminAuthHeaders()) :
getChart(chart.getId(), fields, adminAuthHeaders());
assertNotNull(chart.getOwner());
assertNotNull(chart.getService());
assertNotNull(chart.getServiceType()); // We always return the service
}
public static void getChart(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
@ -272,6 +259,7 @@ public class ChartResourceTest extends EntityResourceTest<Chart> {
CreateChart createRequest = (CreateChart) request;
validateCommonEntityFields(getEntityInterface(chart), createRequest.getDescription(),
TestUtils.getPrincipal(authHeaders), createRequest.getOwner());
assertNotNull(chart.getServiceType());
assertService(createRequest.getService(), chart.getService());
}

20
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java
View File

@ -63,6 +63,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.ENTITY_ALREADY_EXISTS;
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound;
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.invalidServiceEntity;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
@ -156,9 +157,8 @@ public class DashboardResourceTest extends EntityResourceTest<Dashboard> {
CreateDashboard create = create(test).withService(SUPERSET_INVALID_SERVICE_REFERENCE);
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
createDashboard(create, adminAuthHeaders()));
TestUtils.assertResponseContains(exception, BAD_REQUEST, String.format("Invalid service type %s",
SUPERSET_INVALID_SERVICE_REFERENCE.getType()));
TestUtils.assertResponseContains(exception, BAD_REQUEST,
invalidServiceEntity(SUPERSET_INVALID_SERVICE_REFERENCE.getType(), Entity.DASHBOARD));
}
@Test
@ -283,22 +283,17 @@ public class DashboardResourceTest extends EntityResourceTest<Dashboard> {
getDashboard(dashboard.getId(), fields, adminAuthHeaders());
assertNotNull(dashboard.getOwner());
assertNotNull(dashboard.getService()); // We always return the service
assertNotNull(dashboard.getServiceType());
assertNull(dashboard.getCharts());
// .../Dashboards?fields=owner,service
fields = "owner,service";
dashboard = byName ? getDashboardByName(dashboard.getFullyQualifiedName(), fields, adminAuthHeaders()) :
getDashboard(dashboard.getId(), fields, adminAuthHeaders());
assertNotNull(dashboard.getOwner());
assertNotNull(dashboard.getService());
assertNull(dashboard.getCharts());
assertNull(dashboard.getUsageSummary());
// .../Dashboards?fields=owner,service,tables
fields = "owner,service,charts,usageSummary";
dashboard = byName ? getDashboardByName(dashboard.getFullyQualifiedName(), fields, adminAuthHeaders()) :
getDashboard(dashboard.getId(), fields, adminAuthHeaders());
assertNotNull(dashboard.getOwner());
assertNotNull(dashboard.getService());
assertNotNull(dashboard.getService()); // We always return the service
assertNotNull(dashboard.getServiceType());
assertNotNull(dashboard.getCharts());
TestUtils.validateEntityReference(dashboard.getCharts());
assertNotNull(dashboard.getUsageSummary());
@ -361,6 +356,7 @@ public class DashboardResourceTest extends EntityResourceTest<Dashboard> {
CreateDashboard createRequest = (CreateDashboard) request;
validateCommonEntityFields(getEntityInterface(dashboard), createRequest.getDescription(),
TestUtils.getPrincipal(authHeaders), createRequest.getOwner());
assertNotNull(dashboard.getServiceType());
assertService(createRequest.getService(), dashboard.getService());
validateDashboardCharts(dashboard, createRequest.getCharts());
TestUtils.validateTags(createRequest.getTags(), dashboard.getTags());

20
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/DatabaseResourceTest.java
View File

@ -208,22 +208,17 @@ public class DatabaseResourceTest extends EntityResourceTest<Database> {
getDatabase(database.getId(), fields, adminAuthHeaders());
assertNotNull(database.getOwner());
assertNotNull(database.getService()); // We always return the service
assertNotNull(database.getServiceType());
assertNull(database.getTables());
// .../databases?fields=owner,service
fields = "owner,service";
database = byName ? getDatabaseByName(database.getFullyQualifiedName(), fields, adminAuthHeaders()) :
getDatabase(database.getId(), fields, adminAuthHeaders());
assertNotNull(database.getOwner());
assertNotNull(database.getService());
assertNull(database.getTables());
assertNull(database.getUsageSummary());
// .../databases?fields=owner,service,tables
fields = "owner,service,tables,usageSummary";
database = byName ? getDatabaseByName(database.getFullyQualifiedName(), fields, adminAuthHeaders()) :
getDatabase(database.getId(), fields, adminAuthHeaders());
assertNotNull(database.getOwner());
assertNotNull(database.getService());
assertNotNull(database.getService()); // We always return the service
assertNotNull(database.getServiceType());
assertNotNull(database.getTables());
TestUtils.validateEntityReference(database.getTables());
assertNotNull(database.getUsageSummary());
@ -270,13 +265,14 @@ public class DatabaseResourceTest extends EntityResourceTest<Database> {
}
@Override
public void validateCreatedEntity(Database createdEntity, Object request, Map<String, String> authHeaders) {
public void validateCreatedEntity(Database database, Object request, Map<String, String> authHeaders) {
CreateDatabase createRequest = (CreateDatabase) request;
validateCommonEntityFields(getEntityInterface(createdEntity), createRequest.getDescription(),
validateCommonEntityFields(getEntityInterface(database), createRequest.getDescription(),
TestUtils.getPrincipal(authHeaders), createRequest.getOwner());
// Validate service
assertService(createRequest.getService(), createdEntity.getService());
assertNotNull(database.getServiceType());
assertService(createRequest.getService(), database.getService());
}
@Override

11
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java
View File

@ -1084,16 +1084,15 @@ public class TableResourceTest extends EntityResourceTest<Table> {
} else {
assertNull(table.getTableConstraints());
}
if (fields.contains("database")) {
assertNotNull(table.getDatabase());
} else {
assertNull(table.getDatabase());
}
if (fields.contains("tags")) {
assertNotNull(table.getTags());
} else {
assertNull(table.getTags());
}
// Default fields that are always returned
assertNotNull(table.getDatabase());
assertNotNull(table.getService());
assertNotNull(table.getServiceType());
}
/** Validate returned fields GET .../tables/{id}?fields="..." or GET .../tables/name/{fqn}?fields="..." */
@ -1273,6 +1272,8 @@ public class TableResourceTest extends EntityResourceTest<Table> {
assertEquals(createRequest.getTableConstraints(), createdEntity.getTableConstraints());
TestUtils.validateTags(createRequest.getTags(), createdEntity.getTags());
TestUtils.validateEntityReference(createdEntity.getFollowers());
assertNotNull(createdEntity.getService());
assertNotNull(createdEntity.getServiceType());
}
@Override

13
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/pipelines/PipelineResourceTest.java
View File

@ -93,6 +93,7 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline> {
validateCommonEntityFields(getEntityInterface(pipeline), createRequest.getDescription(),
TestUtils.getPrincipal(authHeaders), createRequest.getOwner());
assertEquals(createRequest.getDisplayName(), pipeline.getDisplayName());
assertNotNull(pipeline.getServiceType());
assertService(createRequest.getService(), pipeline.getService());
assertEquals(createRequest.getTasks(), pipeline.getTasks());
TestUtils.validateTags(createRequest.getTags(), pipeline.getTags());
@ -339,14 +340,7 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline> {
getPipeline(pipeline.getId(), fields, adminAuthHeaders());
assertNotNull(pipeline.getOwner());
assertNotNull(pipeline.getService()); // We always return the service
assertNull(pipeline.getTasks());
// .../Pipelines?fields=owner,service
fields = "owner,service";
pipeline = byName ? getPipelineByName(pipeline.getFullyQualifiedName(), fields, adminAuthHeaders()) :
getPipeline(pipeline.getId(), fields, adminAuthHeaders());
assertNotNull(pipeline.getOwner());
assertNotNull(pipeline.getService());
assertNotNull(pipeline.getServiceType());
assertNull(pipeline.getTasks());
// .../Pipelines?fields=owner,service,tables
@ -354,7 +348,8 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline> {
pipeline = byName ? getPipelineByName(pipeline.getFullyQualifiedName(), fields, adminAuthHeaders()) :
getPipeline(pipeline.getId(), fields, adminAuthHeaders());
assertNotNull(pipeline.getOwner());
assertNotNull(pipeline.getService());
assertNotNull(pipeline.getService()); // We always return the service
assertNotNull(pipeline.getServiceType());
assertNotNull(pipeline.getTasks());
}

15
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/topics/TopicResourceTest.java
View File

@ -193,20 +193,7 @@ public class TopicResourceTest extends EntityResourceTest<Topic> {
getTopic(topic.getId(), fields, adminAuthHeaders());
assertNotNull(topic.getOwner());
assertNotNull(topic.getService()); // We always return the service
// .../topics?fields=owner,service
fields = "owner,service";
topic = byName ? getTopicByName(topic.getFullyQualifiedName(), fields, adminAuthHeaders()) :
getTopic(topic.getId(), fields, adminAuthHeaders());
assertNotNull(topic.getOwner());
assertNotNull(topic.getService());
// .../topics?fields=owner,service
fields = "owner,service";
topic = byName ? getTopicByName(topic.getFullyQualifiedName(), fields, adminAuthHeaders()) :
getTopic(topic.getId(), fields, adminAuthHeaders());
assertNotNull(topic.getOwner());
assertNotNull(topic.getService());
assertNotNull(topic.getServiceType());
}
public static void getTopic(UUID id, Map<String, String> authHeaders) throws HttpResponseException {