From 00038449e86bf85225d3189afcd9f9fe8920cfc4 Mon Sep 17 00:00:00 2001 From: sureshms Date: Sat, 27 Nov 2021 10:12:47 -0800 Subject: [PATCH 1/2] Fixes #1149 - Add documentation to EntityUpdater about CRUD mechanism to help implementors understand better --- .../catalog/jdbi3/BotsRepository.java | 6 +- .../catalog/jdbi3/ChartRepository.java | 6 +- .../catalog/jdbi3/DashboardRepository.java | 6 +- .../jdbi3/DashboardServiceRepository.java | 6 +- .../catalog/jdbi3/DatabaseRepository.java | 6 +- .../jdbi3/DatabaseServiceRepository.java | 6 +- .../catalog/jdbi3/DbtModelRepository.java | 6 +- .../catalog/jdbi3/EntityRepository.java | 144 ++++++++++++++---- .../catalog/jdbi3/IngestionRepository.java | 6 +- .../catalog/jdbi3/LocationRepository.java | 6 +- .../jdbi3/MessagingServiceRepository.java | 6 +- .../catalog/jdbi3/MetricsRepository.java | 6 +- .../catalog/jdbi3/MlModelRepository.java | 6 +- .../catalog/jdbi3/PipelineRepository.java | 6 +- .../jdbi3/PipelineServiceRepository.java | 6 +- .../catalog/jdbi3/PolicyRepository.java | 6 +- .../catalog/jdbi3/ReportRepository.java | 6 +- .../jdbi3/StorageServiceRepository.java | 6 +- .../catalog/jdbi3/TableRepository.java | 8 +- .../catalog/jdbi3/TeamRepository.java | 6 +- .../catalog/jdbi3/TopicRepository.java | 6 +- .../catalog/jdbi3/UserRepository.java | 6 +- 22 files changed, 178 insertions(+), 94 deletions(-) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java index 66623ac977c..ed35d228f76 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java @@ -61,15 +61,15 @@ public class BotsRepository extends EntityRepository{ } @Override - public void validate(Bots entity) throws IOException { } + public void prepare(Bots entity) throws IOException { } @Override - public void store(Bots entity, boolean update) throws IOException { + public void storeEntity(Bots entity, boolean update) throws IOException { dao.botsDAO().insert(entity); } @Override - public void storeRelationships(Bots entity) throws IOException { } + public void addRelationships(Bots entity) throws IOException { } public static class BotsEntityInterface implements EntityInterface { private final Bots entity; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java index 5c77df98b0c..320205c6c8f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java @@ -68,7 +68,7 @@ public class ChartRepository extends EntityRepository { } @Override - public void validate(Chart chart) throws IOException { + public void prepare(Chart chart) throws IOException { chart.setService(getService(chart.getService())); chart.setFullyQualifiedName(getFQN(chart)); EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), chart.getOwner()); // Validate owner @@ -76,7 +76,7 @@ public class ChartRepository extends EntityRepository { } @Override - public void store(Chart chart, boolean update) throws JsonProcessingException { + public void storeEntity(Chart chart, boolean update) throws JsonProcessingException { // Relationships and fields such as href are derived and not stored as part of json EntityReference owner = chart.getOwner(); List tags = chart.getTags(); @@ -96,7 +96,7 @@ public class ChartRepository extends EntityRepository { } @Override - public void storeRelationships(Chart chart) throws IOException { + public void addRelationships(Chart chart) throws IOException { EntityReference service = chart.getService(); dao.relationshipDAO().insert(service.getId().toString(), chart.getId().toString(), service.getType(), Entity.CHART, Relationship.CONTAINS.ordinal()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java index cbbb8845639..63a718e10fd 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java @@ -129,7 +129,7 @@ public class DashboardRepository extends EntityRepository { } @Override - public void validate(Dashboard dashboard) throws IOException { + public void prepare(Dashboard dashboard) throws IOException { dashboard.setService(getService(dashboard.getService())); dashboard.setFullyQualifiedName(getFQN(dashboard)); EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), dashboard.getOwner()); // Validate owner @@ -137,7 +137,7 @@ public class DashboardRepository extends EntityRepository { } @Override - public void store(Dashboard dashboard, boolean update) throws JsonProcessingException { + public void storeEntity(Dashboard dashboard, boolean update) throws JsonProcessingException { // Relationships and fields such as href are derived and not stored as part of json EntityReference owner = dashboard.getOwner(); List tags = dashboard.getTags(); @@ -156,7 +156,7 @@ public class DashboardRepository extends EntityRepository { } @Override - public void storeRelationships(Dashboard dashboard) throws IOException { + public void addRelationships(Dashboard dashboard) throws IOException { setService(dashboard, dashboard.getService()); // Add relationship from dashboard to chart diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java index d16fc60172e..3e1e3824da7 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java @@ -87,12 +87,12 @@ public class DashboardServiceRepository extends EntityRepository { } @Override - public void validate(Database database) throws IOException { + public void prepare(Database database) throws IOException { database.setService(getService(database.getService())); database.setFullyQualifiedName(getFQN(database)); database.setOwner(EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), database.getOwner())); // Validate owner } @Override - public void store(Database database, boolean update) throws IOException { + public void storeEntity(Database database, boolean update) throws IOException { // Relationships and fields such as href are derived and not stored as part of json EntityReference owner = database.getOwner(); EntityReference service = database.getService(); @@ -101,7 +101,7 @@ public class DatabaseRepository extends EntityRepository { } @Override - public void storeRelationships(Database database) throws IOException { + public void addRelationships(Database database) throws IOException { dao.relationshipDAO().insert(database.getService().getId().toString(), database.getId().toString(), database.getService().getType(), Entity.DATABASE, Relationship.CONTAINS.ordinal()); EntityUtil.setOwner(dao.relationshipDAO(), database.getId(), Entity.DATABASE, database.getOwner()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java index c12c44698e8..302d0159559 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java @@ -77,12 +77,12 @@ public class DatabaseServiceRepository extends EntityRepository } @Override - public void validate(DatabaseService entity) throws IOException { + public void prepare(DatabaseService entity) throws IOException { EntityUtil.validateIngestionSchedule(entity.getIngestionSchedule()); } @Override - public void store(DatabaseService service, boolean update) throws IOException { + public void storeEntity(DatabaseService service, boolean update) throws IOException { if (update) { dao.dbServiceDAO().update(service.getId(), JsonUtils.pojoToJson(service)); } else { @@ -91,7 +91,7 @@ public class DatabaseServiceRepository extends EntityRepository } @Override - public void storeRelationships(DatabaseService entity) throws IOException { + public void addRelationships(DatabaseService entity) throws IOException { } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java index f28b60ceaac..3842c2987b5 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java @@ -127,7 +127,7 @@ public class DbtModelRepository extends EntityRepository { } @Override - public void validate(DbtModel dbtModel) throws IOException { + public void prepare(DbtModel dbtModel) throws IOException { dbtModel.setDatabase(dao.databaseDAO().findEntityReferenceById(dbtModel.getDatabase().getId())); // Set data in table entity based on database relationship @@ -145,7 +145,7 @@ public class DbtModelRepository extends EntityRepository { } @Override - public void store(DbtModel dbtModel, boolean update) throws IOException { + public void storeEntity(DbtModel dbtModel, boolean update) throws IOException { // Relationships and fields such as href are derived and not stored as part of json EntityReference owner = dbtModel.getOwner(); EntityReference database = dbtModel.getDatabase(); @@ -171,7 +171,7 @@ public class DbtModelRepository extends EntityRepository { } @Override - public void storeRelationships(DbtModel dbtModel) throws IOException { + public void addRelationships(DbtModel dbtModel) throws IOException { // Add relationship from database to model String databaseId = dbtModel.getDatabase().getId().toString(); dao.relationshipDAO().insert(databaseId, dbtModel.getId().toString(), Entity.DATABASE, Entity.DBTMODEL, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java index 82c8ec2e2a3..a04c078972a 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java @@ -19,6 +19,7 @@ package org.openmetadata.catalog.jdbi3; import com.fasterxml.jackson.core.JsonProcessingException; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.entity.data.Table; import org.openmetadata.catalog.entity.teams.User; import org.openmetadata.catalog.exception.CatalogExceptionMessage; import org.openmetadata.catalog.exception.EntityNotFoundException; @@ -60,8 +61,28 @@ import java.util.UUID; import java.util.function.BiPredicate; /** - * Interface used for accessing the concrete entity DAOs such as table, dashboard etc. - * This gives a uniform access so that common boiler plate code can be reduced. + * This class is used by Entity Resources to perform READ and WRITE operations to the backend database to Create, + * Retrieve, Update, and Delete entities. + * + * An entity has two types of fields - `attributes` and `relationships`. The `attributes` are the core properties of + * the entity, example - entity id, name, fullyQualifiedName, columns for a table, etc. The `relationships` are + * an associated between two entities, example - table belongs to a database, table has a tag, user owns a table, etc. + * All relationships are captured using {@code EntityReference}. + * + * Entities are stored as JSON documents in the database. Each entity is stored in a separate table and is accessed + * through a Data Access Object or DAO that corresponds to each of the entity. All DAO objects for an + * entity are available in {@code daoCollection}. + * + * Relationships between entity is stored in a separate table that captures the edge - fromEntity, toEntity, and + * the relationship name. + * + * JSON document of an entity stores only required attributes of an entity. Some attributes such as href + * are not stored and are created on the fly. + * + * Json document of an entity does not store relationships. As an example, JSON document for table entity + * does not store the relationship database which is of type EntityReference. This is always retrieved + * from the the relationship edges when required to ensure, the data stored is consistent and information in + * responses is not stale. */ public abstract class EntityRepository { public static final Logger LOG = LoggerFactory.getLogger(EntityRepository.class); @@ -70,24 +91,13 @@ public abstract class EntityRepository { private final String entityName; private final EntityDAO dao; private final CollectionDAO daoCollection; + + /** Fields that can be updated during PATCH operation */ private final Fields patchFields; + + /** Fields that can be updated during PUT operation */ private final Fields putFields; - /** - * Entity related operations that should be implemented or overridden by entities - */ - public abstract EntityInterface getEntityInterface(T entity); - - public abstract T setFields(T entity, Fields fields) throws IOException, ParseException; - public abstract void restorePatchAttributes(T original, T updated) throws IOException, ParseException; - public abstract void validate(T entity) throws IOException; - public abstract void store(T entity, boolean update) throws IOException; - public abstract void storeRelationships(T entity) throws IOException; - - public EntityUpdater getUpdater(T original, T updated, boolean patchOperation) throws IOException { - return new EntityUpdater(original, updated, patchOperation); - } - EntityRepository(String collectionPath, String entityName, Class entityClass, EntityDAO entityDAO, CollectionDAO collectionDAO, Fields patchFields, Fields putFields) { @@ -101,6 +111,73 @@ public abstract class EntityRepository { Entity.registerEntity(entityName, dao, this); } + /** + * Entity related operations that should be implemented or overridden by entities + */ + public abstract EntityInterface getEntityInterface(T entity); + + /** + * Set the requested fields in an entity. This is used for requesting specific fields in the object during GET + * operations. It is also used during PUT and PATCH operations to set up fields that can be updated. + */ + public abstract T setFields(T entity, Fields fields) throws IOException, ParseException; + + /** + * This method is used for validating an entity to be created during POST, PUT, and PATCH operations and prepare the + * entity with all the required attributes and relationships. + * + * The implementation of this method must perform the following: + *
    + *
  1. Prepare the values for attributes that are not required in the request but can be derived on the server side. + * Example - >FullyQualifiedNames of an entity can be derived from the hierarchy that an entity belongs to + * .
    2. + *
  2. Validate all the attributes of an entity.
    3. + *
  3. Validate all the relationships of an entity. As an example - during table creation, + * relationships such as Tags, Owner, Databasea table belongs to are validated. + * During validation additional information that is not required in the create/update request are set up + * in the corresponding relationship fields.
    4. + *
+ * + * At the end of this operation, entity is expected to be valid and fully constructed with all the fields that will + * be sent as payload in the POST, PUT, and PATCH operations response. + * + * @see TableRepository#prepare(Table) for an example implementation + */ + public abstract void prepare(T entity) throws IOException; + + /** + * An entity is stored in the backend database as JSON document. The JSON includes only some of the attributes of the + * entity and does not include attributes such as href. The relationship fields of an entity is never stored + * in the JSON document. It is always reconstructed based on relationship edges from the backend database. + * + * As an example, when table entity is stored, the attributes such as href and the relationships such + * as owner, database, and tags are set to null. These attributes are restored back after the + * JSON document is stored to be sent as response. + * + * @see TableRepository#storeEntity(Table, boolean) for an example implementation + */ + public abstract void storeEntity(T entity, boolean update) throws IOException; + + /** + * This method is called to store all the relationships of an entity. It is expected that all relationships are + * already validated and completely setup before this method is called and no validation of relationships is required. + * + * @see TableRepository#addRelationships(Table) for an example implementation + */ + public abstract void addRelationships(T entity) throws IOException; + + /** + * PATCH operations can't overwrite certain fields, such as entity ID, fullyQualifiedNames etc. Instead of throwing + * an error, we take lenient approach of ignoring the user error and restore those attributes based on what is + * already stored as original entity. + */ + public abstract void restorePatchAttributes(T original, T updated) throws IOException, ParseException; + + + public EntityUpdater getUpdater(T original, T updated, boolean patchOperation) throws IOException { + return new EntityUpdater(original, updated, patchOperation); + } + @Transaction public final T get(UriInfo uriInfo, String id, Fields fields) throws IOException, ParseException { return withHref(uriInfo, setFields(dao.findEntityById(UUID.fromString(id)), fields)); @@ -195,23 +272,23 @@ public abstract class EntityRepository { @Transaction public final T createInternal(T entity) throws IOException, ParseException { - validate(entity); + prepare(entity); return createNewEntity(entity); } @Transaction public final PutResponse createOrUpdate(UriInfo uriInfo, T updated) throws IOException, ParseException { - validate(updated); + prepare(updated); T original = JsonUtils.readValue(dao.findJsonByFqn(getFullyQualifiedName(updated)), entityClass); if (original == null) { return new PutResponse<>(Status.CREATED, withHref(uriInfo, createNewEntity(updated)), RestUtil.ENTITY_CREATED); } - // Update the existing entity + // Get all the fields in the original entity that can be updated during PUT operation setFields(original, putFields); + // Update the attributes and relationships of an entity EntityUpdater entityUpdater = getUpdater(original, updated, false); entityUpdater.update(); - entityUpdater.store(); String change = entityUpdater.fieldsChanged() ? RestUtil.ENTITY_UPDATED : RestUtil.ENTITY_NO_CHANGE; return new PutResponse<>(Status.OK, withHref(uriInfo, updated), change); } @@ -219,16 +296,20 @@ public abstract class EntityRepository { @Transaction public final PatchResponse patch(UriInfo uriInfo, UUID id, String user, JsonPatch patch) throws IOException, ParseException { + // Get all the fields in the original entity that can be updated during PATCH operation T original = setFields(dao.findEntityById(id), patchFields); + + // Apply JSON patch to the original entity to get the updated entity T updated = JsonUtils.applyPatch(original, patch, entityClass); EntityInterface updatedEntity = getEntityInterface(updated); updatedEntity.setUpdateDetails(user, new Date()); - validate(updated); + prepare(updated); restorePatchAttributes(original, updated); + + // Update the attributes and relationships of an entity EntityUpdater entityUpdater = getUpdater(original, updated, true); entityUpdater.update(); - entityUpdater.store(); String change = entityUpdater.fieldsChanged() ? RestUtil.ENTITY_UPDATED : RestUtil.ENTITY_NO_CHANGE; return new PatchResponse<>(Status.OK, withHref(uriInfo, updated), change); } @@ -239,7 +320,7 @@ public abstract class EntityRepository { T entity = dao.findEntityById(entityId); EntityInterface entityInterface = getEntityInterface(entity); - // Validate user + // Validate follower User user = daoCollection.userDAO().findEntityById(userId); if (user.getDeactivated()) { throw new IllegalArgumentException(CatalogExceptionMessage.deactivatedUser(userId)); @@ -266,7 +347,7 @@ public abstract class EntityRepository { T entity = dao.findEntityById(entityId); EntityInterface entityInterface = getEntityInterface(entity); - // Validate user + // Validate follower User user = daoCollection.userDAO().findEntityById(userId); // Remove follower @@ -294,8 +375,8 @@ public abstract class EntityRepository { } private T createNewEntity(T entity) throws IOException { - store(entity, false); - storeRelationships(entity); + storeEntity(entity, false); + addRelationships(entity); return entity; } @@ -312,7 +393,7 @@ public abstract class EntityRepository { } /** - * Class that performs PUT and PATCH UPDATE operation. Override {@code entitySpecificUpdate()} to add + * Class that performs PUT and PATCH update operation. Override {@code entitySpecificUpdate()} to add * additional entity specific fields to be updated. */ public class EntityUpdater { @@ -328,13 +409,16 @@ public abstract class EntityRepository { this.patchOperation = patchOperation; } - public final void update() throws IOException { + public final void update() throws IOException, ParseException { updated.setId(original.getId()); updateDescription(); updateDisplayName(); updateOwner(); updateTags(updated.getFullyQualifiedName(), "tags", original.getTags(), updated.getTags()); entitySpecificUpdate(); + + // Store the updated entity + store(); } public void entitySpecificUpdate() throws IOException { @@ -488,7 +572,7 @@ public abstract class EntityRepository { JsonUtils.pojoToJson(original.getEntity())); // Store the new version - EntityRepository.this.store(updated.getEntity(), true); + EntityRepository.this.storeEntity(updated.getEntity(), true); } else { updated.setUpdateDetails(original.getUpdatedBy(), original.getUpdatedAt()); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java index 287226cc7ee..37bf4daf9ea 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java @@ -100,7 +100,7 @@ public class IngestionRepository extends EntityRepository { @Override - public void validate(Ingestion ingestion) throws IOException { + public void prepare(Ingestion ingestion) throws IOException { ingestion.setService(getService(ingestion.getService())); ingestion.setFullyQualifiedName(getFQN(ingestion)); EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), ingestion.getOwner()); // Validate owner @@ -109,7 +109,7 @@ public class IngestionRepository extends EntityRepository { } @Override - public void store(Ingestion ingestion, boolean update) throws IOException { + public void storeEntity(Ingestion ingestion, boolean update) throws IOException { // Relationships and fields such as href are derived and not stored as part of json EntityReference owner = ingestion.getOwner(); List tags = ingestion.getTags(); @@ -129,7 +129,7 @@ public class IngestionRepository extends EntityRepository { } @Override - public void storeRelationships(Ingestion ingestion) throws IOException { + public void addRelationships(Ingestion ingestion) throws IOException { EntityReference service = ingestion.getService(); dao.relationshipDAO().insert(service.getId().toString(), ingestion.getId().toString(), service.getType(), Entity.INGESTION, Relationship.CONTAINS.ordinal()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java index 162f9d3cee8..a61527a7351 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java @@ -145,7 +145,7 @@ public class LocationRepository extends EntityRepository { } @Override - public void validate(Location location) throws IOException { + public void prepare(Location location) throws IOException { // Set data in location entity based on storage relationship location.setFullyQualifiedName(getFQN(location)); @@ -157,7 +157,7 @@ public class LocationRepository extends EntityRepository { } @Override - public void store(Location location, boolean update) throws IOException { + public void storeEntity(Location location, boolean update) throws IOException { // Relationships and fields such as href are derived and not stored as part of json EntityReference owner = location.getOwner(); EntityReference service = location.getService(); @@ -177,7 +177,7 @@ public class LocationRepository extends EntityRepository { } @Override - public void storeRelationships(Location location) throws IOException { + public void addRelationships(Location location) throws IOException { // Add location owner relationship EntityUtil.setOwner(dao.relationshipDAO(), location.getId(), Entity.LOCATION, location.getOwner()); dao.relationshipDAO().insert(location.getService().getId().toString(), location.getId().toString(), diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java index 205bed0735e..91b93e9472d 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java @@ -75,12 +75,12 @@ public class MessagingServiceRepository extends EntityRepository { } @Override - public void validate(Metrics metrics) throws IOException { + public void prepare(Metrics metrics) throws IOException { metrics.setFullyQualifiedName(getFQN(metrics)); EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), metrics.getOwner()); // Validate owner metrics.setService(getService(metrics.getService())); @@ -76,7 +76,7 @@ public class MetricsRepository extends EntityRepository { } @Override - public void store(Metrics metrics, boolean update) throws IOException { + public void storeEntity(Metrics metrics, boolean update) throws IOException { // Relationships and fields such as href are derived and not stored as part of json EntityReference owner = metrics.getOwner(); List tags = metrics.getTags(); @@ -96,7 +96,7 @@ public class MetricsRepository extends EntityRepository { } @Override - public void storeRelationships(Metrics metrics) throws IOException { + public void addRelationships(Metrics metrics) throws IOException { dao.relationshipDAO().insert(metrics.getService().getId().toString(), metrics.getId().toString(), metrics.getService().getType(), Entity.METRICS, Relationship.CONTAINS.ordinal()); setOwner(metrics, metrics.getOwner()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java index 64ed29bed9b..42f79ad5012 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java @@ -119,7 +119,7 @@ public class MlModelRepository extends EntityRepository { @Override - public void validate(MlModel mlModel) throws IOException { + public void prepare(MlModel mlModel) throws IOException { mlModel.setFullyQualifiedName(getFQN(mlModel)); if (mlModel.getMlFeatures() != null && !mlModel.getMlFeatures().isEmpty()) { @@ -140,7 +140,7 @@ public class MlModelRepository extends EntityRepository { } @Override - public void store(MlModel mlModel, boolean update) throws IOException { + public void storeEntity(MlModel mlModel, boolean update) throws IOException { // Relationships and fields such as href are derived and not stored as part of json EntityReference owner = mlModel.getOwner(); List tags = mlModel.getTags(); @@ -160,7 +160,7 @@ public class MlModelRepository extends EntityRepository { } @Override - public void storeRelationships(MlModel mlModel) throws IOException { + public void addRelationships(MlModel mlModel) throws IOException { EntityUtil.setOwner(dao.relationshipDAO(), mlModel.getId(), Entity.MLMODEL, mlModel.getOwner()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java index 5f1ac2610a9..c7dac0ba3e6 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java @@ -114,7 +114,7 @@ public class PipelineRepository extends EntityRepository { @Override - public void validate(Pipeline pipeline) throws IOException { + public void prepare(Pipeline pipeline) throws IOException { pipeline.setService(getService(pipeline.getService())); pipeline.setFullyQualifiedName(getFQN(pipeline)); EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), pipeline.getOwner()); // Validate owner @@ -123,7 +123,7 @@ public class PipelineRepository extends EntityRepository { } @Override - public void store(Pipeline pipeline, boolean update) throws IOException { + public void storeEntity(Pipeline pipeline, boolean update) throws IOException { // Relationships and fields such as href are derived and not stored as part of json EntityReference owner = pipeline.getOwner(); List tags = pipeline.getTags(); @@ -143,7 +143,7 @@ public class PipelineRepository extends EntityRepository { } @Override - public void storeRelationships(Pipeline pipeline) throws IOException { + public void addRelationships(Pipeline pipeline) throws IOException { EntityReference service = pipeline.getService(); dao.relationshipDAO().insert(service.getId().toString(), pipeline.getId().toString(), service.getType(), Entity.PIPELINE, Relationship.CONTAINS.ordinal()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java index f5e9c5c634e..080a9ec107b 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java @@ -75,12 +75,12 @@ public class PipelineServiceRepository extends EntityRepository } @Override - public void validate(PipelineService entity) throws IOException { + public void prepare(PipelineService entity) throws IOException { EntityUtil.validateIngestionSchedule(entity.getIngestionSchedule()); } @Override - public void store(PipelineService service, boolean update) throws IOException { + public void storeEntity(PipelineService service, boolean update) throws IOException { if (update) { dao.pipelineServiceDAO().update(service.getId(), JsonUtils.pojoToJson(service)); } else { @@ -89,7 +89,7 @@ public class PipelineServiceRepository extends EntityRepository } @Override - public void storeRelationships(PipelineService entity) throws IOException { + public void addRelationships(PipelineService entity) throws IOException { } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java index 459ab363e27..bc3209a8e5e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java @@ -95,7 +95,7 @@ public class PolicyRepository extends EntityRepository { @Override - public void validate(Policy policy) throws IOException { + public void prepare(Policy policy) throws IOException { policy.setFullyQualifiedName(getFQN(policy)); // Check if owner is valid and set the relationship @@ -103,7 +103,7 @@ public class PolicyRepository extends EntityRepository { } @Override - public void store(Policy policy, boolean update) throws IOException { + public void storeEntity(Policy policy, boolean update) throws IOException { // Relationships and fields such as href are derived and not stored as part of json EntityReference owner = policy.getOwner(); URI href = policy.getHref(); @@ -122,7 +122,7 @@ public class PolicyRepository extends EntityRepository { } @Override - public void storeRelationships(Policy policy) throws IOException { + public void addRelationships(Policy policy) throws IOException { // Add policy owner relationship setOwner(policy, policy.getOwner()); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java index 90cc66b0290..9c8c439004c 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java @@ -63,20 +63,20 @@ public class ReportRepository extends EntityRepository { } @Override - public void validate(Report report) throws IOException { + public void prepare(Report report) throws IOException { setService(report, report.getService()); setOwner(report, report.getOwner()); EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), report.getOwner()); // Validate owner } @Override - public void store(Report report, boolean update) throws IOException { + public void storeEntity(Report report, boolean update) throws IOException { // TODO add right checks dao.reportDAO().insert(report); } @Override - public void storeRelationships(Report entity) throws IOException { + public void addRelationships(Report entity) throws IOException { // TODO } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java index 34d0b90148e..1d2a8f318ac 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java @@ -70,12 +70,12 @@ public class StorageServiceRepository extends EntityRepository { } @Override - public void validate(StorageService entity) throws IOException { + public void prepare(StorageService entity) throws IOException { } @Override - public void store(StorageService service, boolean update) throws IOException { + public void storeEntity(StorageService service, boolean update) throws IOException { if (update) { dao.storageServiceDAO().update(service.getId(), JsonUtils.pojoToJson(service)); } else { @@ -84,7 +84,7 @@ public class StorageServiceRepository extends EntityRepository { } @Override - public void storeRelationships(StorageService entity) throws IOException { + public void addRelationships(StorageService entity) throws IOException { } public static class StorageServiceEntityInterface implements EntityInterface { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java index 0be1f692a3a..8803806ed0b 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java @@ -108,7 +108,7 @@ public class TableRepository extends EntityRepository { @Override public void restorePatchAttributes(Table original, Table updated) throws IOException, ParseException { - // Patch can't make changes to following fields. Ignore the changes + // Patch can't make changes to following fields. Ignore the changes. updated.withFullyQualifiedName(original.getFullyQualifiedName()).withName(original.getName()) .withDatabase(original.getDatabase()).withId(original.getId()); } @@ -260,7 +260,7 @@ public class TableRepository extends EntityRepository
{ } @Override - public void validate(Table table) throws IOException { + public void prepare(Table table) throws IOException { table.setDatabase(dao.databaseDAO().findEntityReferenceById(table.getDatabase().getId())); // Set data in table entity based on database relationship @@ -278,7 +278,7 @@ public class TableRepository extends EntityRepository
{ } @Override - public void store(Table table, boolean update) throws IOException { + public void storeEntity(Table table, boolean update) throws IOException { // Relationships and fields such as href are derived and not stored as part of json EntityReference owner = table.getOwner(); EntityReference database = table.getDatabase(); @@ -304,7 +304,7 @@ public class TableRepository extends EntityRepository
{ } @Override - public void storeRelationships(Table table) throws IOException { + public void addRelationships(Table table) throws IOException { // Add relationship from database to table String databaseId = table.getDatabase().getId().toString(); dao.relationshipDAO().insert(databaseId, table.getId().toString(), Entity.DATABASE, Entity.TABLE, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java index 7a43b185ce1..9595e216b8a 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java @@ -107,12 +107,12 @@ public class TeamRepository extends EntityRepository { } @Override - public void validate(Team team) throws IOException { + public void prepare(Team team) throws IOException { validateUsers(team.getUsers()); } @Override - public void store(Team team, boolean update) throws IOException { + public void storeEntity(Team team, boolean update) throws IOException { // Relationships and fields such as href are derived and not stored as part of json List users = team.getUsers(); @@ -130,7 +130,7 @@ public class TeamRepository extends EntityRepository { } @Override - public void storeRelationships(Team team) throws IOException { + public void addRelationships(Team team) throws IOException { for (EntityReference user : Optional.ofNullable(team.getUsers()).orElse(Collections.emptyList())) { dao.relationshipDAO().insert(team.getId().toString(), user.getId().toString(), "team", "user", Relationship.CONTAINS.ordinal()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java index 240f510eebb..8f04ed8a5b2 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java @@ -72,7 +72,7 @@ public class TopicRepository extends EntityRepository { } @Override - public void validate(Topic topic) throws IOException { + public void prepare(Topic topic) throws IOException { EntityReference messagingService = getService(topic.getService()); topic.setService(messagingService); topic.setFullyQualifiedName(getFQN(topic)); @@ -81,7 +81,7 @@ public class TopicRepository extends EntityRepository { } @Override - public void store(Topic topic, boolean update) throws IOException { + public void storeEntity(Topic topic, boolean update) throws IOException { // Relationships and fields such as href are derived and not stored as part of json EntityReference owner = topic.getOwner(); List tags = topic.getTags(); @@ -101,7 +101,7 @@ public class TopicRepository extends EntityRepository { } @Override - public void storeRelationships(Topic topic) throws IOException { + public void addRelationships(Topic topic) throws IOException { setService(topic, topic.getService()); setOwner(topic, topic.getOwner()); applyTags(topic); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java index cd129f1ec97..00548429315 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java @@ -67,12 +67,12 @@ public class UserRepository extends EntityRepository { @Override - public void validate(User entity) throws IOException { + public void prepare(User entity) throws IOException { } @Override - public void store(User user, boolean update) throws IOException { + public void storeEntity(User user, boolean update) throws IOException { // Relationships and fields such as href are derived and not stored as part of json List teams = user.getTeams(); @@ -90,7 +90,7 @@ public class UserRepository extends EntityRepository { } @Override - public void storeRelationships(User user) throws IOException { + public void addRelationships(User user) throws IOException { assignTeams(user, user.getTeams()); } From 93a644dfcd80850628dadb4fe2d83d1d479809a6 Mon Sep 17 00:00:00 2001 From: sureshms Date: Sun, 28 Nov 2021 12:02:52 -0800 Subject: [PATCH 2/2] Fixes #1149 Document EntityRepository to help contributors understand how to use it to add new entities --- .../catalog/jdbi3/BotsRepository.java | 2 +- .../catalog/jdbi3/ChartRepository.java | 2 +- .../catalog/jdbi3/DashboardRepository.java | 2 +- .../jdbi3/DashboardServiceRepository.java | 2 +- .../catalog/jdbi3/DatabaseRepository.java | 2 +- .../jdbi3/DatabaseServiceRepository.java | 2 +- .../catalog/jdbi3/DbtModelRepository.java | 2 +- .../catalog/jdbi3/EntityRepository.java | 62 +++++++++++++------ .../catalog/jdbi3/IngestionRepository.java | 2 +- .../catalog/jdbi3/LocationRepository.java | 2 +- .../jdbi3/MessagingServiceRepository.java | 2 +- .../catalog/jdbi3/MetricsRepository.java | 2 +- .../catalog/jdbi3/MlModelRepository.java | 2 +- .../catalog/jdbi3/PipelineRepository.java | 2 +- .../jdbi3/PipelineServiceRepository.java | 2 +- .../catalog/jdbi3/PolicyRepository.java | 2 +- .../catalog/jdbi3/ReportRepository.java | 2 +- .../jdbi3/StorageServiceRepository.java | 2 +- .../catalog/jdbi3/TableRepository.java | 2 +- .../catalog/jdbi3/TeamRepository.java | 2 +- .../catalog/jdbi3/TopicRepository.java | 2 +- .../catalog/jdbi3/UserRepository.java | 2 +- 22 files changed, 64 insertions(+), 40 deletions(-) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java index ed35d228f76..5d17ccfcaa7 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java @@ -69,7 +69,7 @@ public class BotsRepository extends EntityRepository{ } @Override - public void addRelationships(Bots entity) throws IOException { } + public void storeRelationships(Bots entity) throws IOException { } public static class BotsEntityInterface implements EntityInterface { private final Bots entity; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java index 320205c6c8f..43e86064e42 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java @@ -96,7 +96,7 @@ public class ChartRepository extends EntityRepository { } @Override - public void addRelationships(Chart chart) throws IOException { + public void storeRelationships(Chart chart) throws IOException { EntityReference service = chart.getService(); dao.relationshipDAO().insert(service.getId().toString(), chart.getId().toString(), service.getType(), Entity.CHART, Relationship.CONTAINS.ordinal()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java index 63a718e10fd..67b80f10c90 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java @@ -156,7 +156,7 @@ public class DashboardRepository extends EntityRepository { } @Override - public void addRelationships(Dashboard dashboard) throws IOException { + public void storeRelationships(Dashboard dashboard) throws IOException { setService(dashboard, dashboard.getService()); // Add relationship from dashboard to chart diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java index 3e1e3824da7..1254915545d 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java @@ -101,7 +101,7 @@ public class DashboardServiceRepository extends EntityRepository { } @Override - public void addRelationships(Database database) throws IOException { + public void storeRelationships(Database database) throws IOException { dao.relationshipDAO().insert(database.getService().getId().toString(), database.getId().toString(), database.getService().getType(), Entity.DATABASE, Relationship.CONTAINS.ordinal()); EntityUtil.setOwner(dao.relationshipDAO(), database.getId(), Entity.DATABASE, database.getOwner()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java index 302d0159559..6ba0e801592 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java @@ -91,7 +91,7 @@ public class DatabaseServiceRepository extends EntityRepository } @Override - public void addRelationships(DatabaseService entity) throws IOException { + public void storeRelationships(DatabaseService entity) throws IOException { } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java index 3842c2987b5..4388b475785 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java @@ -171,7 +171,7 @@ public class DbtModelRepository extends EntityRepository { } @Override - public void addRelationships(DbtModel dbtModel) throws IOException { + public void storeRelationships(DbtModel dbtModel) throws IOException { // Add relationship from database to model String databaseId = dbtModel.getDatabase().getId().toString(); dao.relationshipDAO().insert(databaseId, dbtModel.getId().toString(), Entity.DATABASE, Entity.DBTMODEL, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java index a04c078972a..742d0f5f854 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java @@ -24,6 +24,7 @@ import org.openmetadata.catalog.entity.teams.User; import org.openmetadata.catalog.exception.CatalogExceptionMessage; import org.openmetadata.catalog.exception.EntityNotFoundException; import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityVersionPair; +import org.openmetadata.catalog.jdbi3.TableRepository.TableUpdater; import org.openmetadata.catalog.type.ChangeDescription; import org.openmetadata.catalog.type.ChangeEvent; import org.openmetadata.catalog.type.EntityHistory; @@ -61,28 +62,39 @@ import java.util.UUID; import java.util.function.BiPredicate; /** - * This class is used by Entity Resources to perform READ and WRITE operations to the backend database to Create, - * Retrieve, Update, and Delete entities. + * This is the base class used by Entity Resources to perform READ and WRITE operations to the backend database to + * Create, * Retrieve, Update, and Delete entities. * - * An entity has two types of fields - `attributes` and `relationships`. The `attributes` are the core properties of - * the entity, example - entity id, name, fullyQualifiedName, columns for a table, etc. The `relationships` are - * an associated between two entities, example - table belongs to a database, table has a tag, user owns a table, etc. - * All relationships are captured using {@code EntityReference}. + * An entity has two types of fields - `attributes` and `relationships`. + *
    + *
  • The `attributes` are the core properties of the entity, example - entity id, name, fullyQualifiedName, columns + * for a table, etc.
    • + *
  • The `relationships` are an associated between two entities, example - table belongs to a database, + * table has a tag, user owns a table, etc. All relationships are captured using {@code EntityReference}.
    • + *
* * Entities are stored as JSON documents in the database. Each entity is stored in a separate table and is accessed - * through a Data Access Object or DAO that corresponds to each of the entity. All DAO objects for an - * entity are available in {@code daoCollection}. + * through a Data Access Object or DAO that corresponds to each of the entity. For example, + * table_entity is the database table used to store JSON docs corresponding to table entity and + * {@link org.openmetadata.catalog.jdbi3.CollectionDAO.TableDAO} is used as the DAO object to access the table_entity + * table. All DAO objects for an entity are available in {@code daoCollection}. * + *

* Relationships between entity is stored in a separate table that captures the edge - fromEntity, toEntity, and - * the relationship name. + * the relationship name entity_relationship table and are supported by + * {@link org.openmetadata.catalog.jdbi3.CollectionDAO.EntityRelationshipDAO} DAO object. * - * JSON document of an entity stores only required attributes of an entity. Some attributes such as href - * are not stored and are created on the fly. + * JSON document of an entity stores only required attributes of an entity. Some attributes such as + * href are not stored and are created on the fly. + * + *

* * Json document of an entity does not store relationships. As an example, JSON document for table entity * does not store the relationship database which is of type EntityReference. This is always retrieved - * from the the relationship edges when required to ensure, the data stored is consistent and information in - * responses is not stale. + * from the the relationship table when required to ensure, the data stored is efficiently and consistently, and + * relationship information does not become stale. + *

+ * */ public abstract class EntityRepository { public static final Logger LOG = LoggerFactory.getLogger(EntityRepository.class); @@ -150,6 +162,8 @@ public abstract class EntityRepository { * entity and does not include attributes such as href. The relationship fields of an entity is never stored * in the JSON document. It is always reconstructed based on relationship edges from the backend database. * + *

+ * * As an example, when table entity is stored, the attributes such as href and the relationships such * as owner, database, and tags are set to null. These attributes are restored back after the * JSON document is stored to be sent as response. @@ -162,14 +176,14 @@ public abstract class EntityRepository { * This method is called to store all the relationships of an entity. It is expected that all relationships are * already validated and completely setup before this method is called and no validation of relationships is required. * - * @see TableRepository#addRelationships(Table) for an example implementation + * @see TableRepository#storeRelationships(Table) for an example implementation */ - public abstract void addRelationships(T entity) throws IOException; + public abstract void storeRelationships(T entity) throws IOException; /** * PATCH operations can't overwrite certain fields, such as entity ID, fullyQualifiedNames etc. Instead of throwing * an error, we take lenient approach of ignoring the user error and restore those attributes based on what is - * already stored as original entity. + * already stored in the original entity. */ public abstract void restorePatchAttributes(T original, T updated) throws IOException, ParseException; @@ -376,7 +390,7 @@ public abstract class EntityRepository { private T createNewEntity(T entity) throws IOException { storeEntity(entity, false); - addRelationships(entity); + storeRelationships(entity); return entity; } @@ -393,8 +407,15 @@ public abstract class EntityRepository { } /** - * Class that performs PUT and PATCH update operation. Override {@code entitySpecificUpdate()} to add - * additional entity specific fields to be updated. + * Class that performs PUT and PATCH update operation. It takes an updated entity and original entity. + * Performs comparison between then and updates the stored entity and also updates all the relationships. This class + * also tracks the changes between original and updated to version the entity and produce change events. + * + *

+ * + * Common entity attributes such as description, displayName, owner, tags are handled by this class. + * Override {@code entitySpecificUpdate()} to add additional entity specific fields to be updated. + * @see TableUpdater#entitySpecificUpdate() for example. */ public class EntityUpdater { protected final EntityInterface original; @@ -409,6 +430,9 @@ public abstract class EntityRepository { this.patchOperation = patchOperation; } + /** + * Compare original and updated entities and perform updates. Update the entity version and track changes. + */ public final void update() throws IOException, ParseException { updated.setId(original.getId()); updateDescription(); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java index 37bf4daf9ea..274535c4496 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java @@ -129,7 +129,7 @@ public class IngestionRepository extends EntityRepository { } @Override - public void addRelationships(Ingestion ingestion) throws IOException { + public void storeRelationships(Ingestion ingestion) throws IOException { EntityReference service = ingestion.getService(); dao.relationshipDAO().insert(service.getId().toString(), ingestion.getId().toString(), service.getType(), Entity.INGESTION, Relationship.CONTAINS.ordinal()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java index a61527a7351..64965c23f9d 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java @@ -177,7 +177,7 @@ public class LocationRepository extends EntityRepository { } @Override - public void addRelationships(Location location) throws IOException { + public void storeRelationships(Location location) throws IOException { // Add location owner relationship EntityUtil.setOwner(dao.relationshipDAO(), location.getId(), Entity.LOCATION, location.getOwner()); dao.relationshipDAO().insert(location.getService().getId().toString(), location.getId().toString(), diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java index 91b93e9472d..32c4e0def77 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java @@ -89,7 +89,7 @@ public class MessagingServiceRepository extends EntityRepository { } @Override - public void addRelationships(Metrics metrics) throws IOException { + public void storeRelationships(Metrics metrics) throws IOException { dao.relationshipDAO().insert(metrics.getService().getId().toString(), metrics.getId().toString(), metrics.getService().getType(), Entity.METRICS, Relationship.CONTAINS.ordinal()); setOwner(metrics, metrics.getOwner()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java index 42f79ad5012..0d8ac7156c6 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java @@ -160,7 +160,7 @@ public class MlModelRepository extends EntityRepository { } @Override - public void addRelationships(MlModel mlModel) throws IOException { + public void storeRelationships(MlModel mlModel) throws IOException { EntityUtil.setOwner(dao.relationshipDAO(), mlModel.getId(), Entity.MLMODEL, mlModel.getOwner()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java index c7dac0ba3e6..00b61c8f5b4 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java @@ -143,7 +143,7 @@ public class PipelineRepository extends EntityRepository { } @Override - public void addRelationships(Pipeline pipeline) throws IOException { + public void storeRelationships(Pipeline pipeline) throws IOException { EntityReference service = pipeline.getService(); dao.relationshipDAO().insert(service.getId().toString(), pipeline.getId().toString(), service.getType(), Entity.PIPELINE, Relationship.CONTAINS.ordinal()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java index 080a9ec107b..3e1d47841c4 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java @@ -89,7 +89,7 @@ public class PipelineServiceRepository extends EntityRepository } @Override - public void addRelationships(PipelineService entity) throws IOException { + public void storeRelationships(PipelineService entity) throws IOException { } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java index bc3209a8e5e..1ee3cdbb367 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java @@ -122,7 +122,7 @@ public class PolicyRepository extends EntityRepository { } @Override - public void addRelationships(Policy policy) throws IOException { + public void storeRelationships(Policy policy) throws IOException { // Add policy owner relationship setOwner(policy, policy.getOwner()); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java index 9c8c439004c..21fd5342b7e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java @@ -76,7 +76,7 @@ public class ReportRepository extends EntityRepository { } @Override - public void addRelationships(Report entity) throws IOException { + public void storeRelationships(Report entity) throws IOException { // TODO } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java index 1d2a8f318ac..dfa8a8c74b4 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java @@ -84,7 +84,7 @@ public class StorageServiceRepository extends EntityRepository { } @Override - public void addRelationships(StorageService entity) throws IOException { + public void storeRelationships(StorageService entity) throws IOException { } public static class StorageServiceEntityInterface implements EntityInterface { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java index 8803806ed0b..bcc78981e0f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java @@ -304,7 +304,7 @@ public class TableRepository extends EntityRepository
{ } @Override - public void addRelationships(Table table) throws IOException { + public void storeRelationships(Table table) throws IOException { // Add relationship from database to table String databaseId = table.getDatabase().getId().toString(); dao.relationshipDAO().insert(databaseId, table.getId().toString(), Entity.DATABASE, Entity.TABLE, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java index 9595e216b8a..365a940a524 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java @@ -130,7 +130,7 @@ public class TeamRepository extends EntityRepository { } @Override - public void addRelationships(Team team) throws IOException { + public void storeRelationships(Team team) throws IOException { for (EntityReference user : Optional.ofNullable(team.getUsers()).orElse(Collections.emptyList())) { dao.relationshipDAO().insert(team.getId().toString(), user.getId().toString(), "team", "user", Relationship.CONTAINS.ordinal()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java index 8f04ed8a5b2..286bc708463 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java @@ -101,7 +101,7 @@ public class TopicRepository extends EntityRepository { } @Override - public void addRelationships(Topic topic) throws IOException { + public void storeRelationships(Topic topic) throws IOException { setService(topic, topic.getService()); setOwner(topic, topic.getOwner()); applyTags(topic); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java index 00548429315..5a348d5f774 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java @@ -90,7 +90,7 @@ public class UserRepository extends EntityRepository { } @Override - public void addRelationships(User user) throws IOException { + public void storeRelationships(User user) throws IOException { assignTeams(user, user.getTeams()); }