diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java index ed35d228f76..5d17ccfcaa7 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java @@ -69,7 +69,7 @@ public class BotsRepository extends EntityRepository{ } @Override - public void addRelationships(Bots entity) throws IOException { } + public void storeRelationships(Bots entity) throws IOException { } public static class BotsEntityInterface implements EntityInterface { private final Bots entity; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java index 320205c6c8f..43e86064e42 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java @@ -96,7 +96,7 @@ public class ChartRepository extends EntityRepository { } @Override - public void addRelationships(Chart chart) throws IOException { + public void storeRelationships(Chart chart) throws IOException { EntityReference service = chart.getService(); dao.relationshipDAO().insert(service.getId().toString(), chart.getId().toString(), service.getType(), Entity.CHART, Relationship.CONTAINS.ordinal()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java index 63a718e10fd..67b80f10c90 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java @@ -156,7 +156,7 @@ public class DashboardRepository extends EntityRepository { } @Override - public void addRelationships(Dashboard dashboard) throws IOException { + public void storeRelationships(Dashboard dashboard) throws IOException { setService(dashboard, dashboard.getService()); // Add relationship from dashboard to chart diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java index 3e1e3824da7..1254915545d 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java @@ -101,7 +101,7 @@ public class DashboardServiceRepository extends EntityRepository { } @Override - public void addRelationships(Database database) throws IOException { + public void storeRelationships(Database database) throws IOException { dao.relationshipDAO().insert(database.getService().getId().toString(), database.getId().toString(), database.getService().getType(), Entity.DATABASE, Relationship.CONTAINS.ordinal()); EntityUtil.setOwner(dao.relationshipDAO(), database.getId(), Entity.DATABASE, database.getOwner()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java index 302d0159559..6ba0e801592 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java @@ -91,7 +91,7 @@ public class DatabaseServiceRepository extends EntityRepository } @Override - public void addRelationships(DatabaseService entity) throws IOException { + public void storeRelationships(DatabaseService entity) throws IOException { } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java index 3842c2987b5..4388b475785 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DbtModelRepository.java @@ -171,7 +171,7 @@ public class DbtModelRepository extends EntityRepository { } @Override - public void addRelationships(DbtModel dbtModel) throws IOException { + public void storeRelationships(DbtModel dbtModel) throws IOException { // Add relationship from database to model String databaseId = dbtModel.getDatabase().getId().toString(); dao.relationshipDAO().insert(databaseId, dbtModel.getId().toString(), Entity.DATABASE, Entity.DBTMODEL, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java index a04c078972a..742d0f5f854 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java @@ -24,6 +24,7 @@ import org.openmetadata.catalog.entity.teams.User; import org.openmetadata.catalog.exception.CatalogExceptionMessage; import org.openmetadata.catalog.exception.EntityNotFoundException; import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityVersionPair; +import org.openmetadata.catalog.jdbi3.TableRepository.TableUpdater; import org.openmetadata.catalog.type.ChangeDescription; import org.openmetadata.catalog.type.ChangeEvent; import org.openmetadata.catalog.type.EntityHistory; @@ -61,28 +62,39 @@ import java.util.UUID; import java.util.function.BiPredicate; /** - * This class is used by Entity Resources to perform READ and WRITE operations to the backend database to Create, - * Retrieve, Update, and Delete entities. + * This is the base class used by Entity Resources to perform READ and WRITE operations to the backend database to + * Create, * Retrieve, Update, and Delete entities. * - * An entity has two types of fields - `attributes` and `relationships`. The `attributes` are the core properties of - * the entity, example - entity id, name, fullyQualifiedName, columns for a table, etc. The `relationships` are - * an associated between two entities, example - table belongs to a database, table has a tag, user owns a table, etc. - * All relationships are captured using {@code EntityReference}. + * An entity has two types of fields - `attributes` and `relationships`. + *
    + *
  • The `attributes` are the core properties of the entity, example - entity id, name, fullyQualifiedName, columns + * for a table, etc.
  • + *
  • The `relationships` are an associated between two entities, example - table belongs to a database, + * table has a tag, user owns a table, etc. All relationships are captured using {@code EntityReference}.
  • + *
* * Entities are stored as JSON documents in the database. Each entity is stored in a separate table and is accessed - * through a Data Access Object or DAO that corresponds to each of the entity. All DAO objects for an - * entity are available in {@code daoCollection}. + * through a Data Access Object or DAO that corresponds to each of the entity. For example, + * table_entity is the database table used to store JSON docs corresponding to table entity and + * {@link org.openmetadata.catalog.jdbi3.CollectionDAO.TableDAO} is used as the DAO object to access the table_entity + * table. All DAO objects for an entity are available in {@code daoCollection}. * + *

* Relationships between entity is stored in a separate table that captures the edge - fromEntity, toEntity, and - * the relationship name. + * the relationship name entity_relationship table and are supported by + * {@link org.openmetadata.catalog.jdbi3.CollectionDAO.EntityRelationshipDAO} DAO object. * - * JSON document of an entity stores only required attributes of an entity. Some attributes such as href - * are not stored and are created on the fly. + * JSON document of an entity stores only required attributes of an entity. Some attributes such as + * href are not stored and are created on the fly. + * + *

* * Json document of an entity does not store relationships. As an example, JSON document for table entity * does not store the relationship database which is of type EntityReference. This is always retrieved - * from the the relationship edges when required to ensure, the data stored is consistent and information in - * responses is not stale. + * from the the relationship table when required to ensure, the data stored is efficiently and consistently, and + * relationship information does not become stale. + *

+ * */ public abstract class EntityRepository { public static final Logger LOG = LoggerFactory.getLogger(EntityRepository.class); @@ -150,6 +162,8 @@ public abstract class EntityRepository { * entity and does not include attributes such as href. The relationship fields of an entity is never stored * in the JSON document. It is always reconstructed based on relationship edges from the backend database. * + *

+ * * As an example, when table entity is stored, the attributes such as href and the relationships such * as owner, database, and tags are set to null. These attributes are restored back after the * JSON document is stored to be sent as response. @@ -162,14 +176,14 @@ public abstract class EntityRepository { * This method is called to store all the relationships of an entity. It is expected that all relationships are * already validated and completely setup before this method is called and no validation of relationships is required. * - * @see TableRepository#addRelationships(Table) for an example implementation + * @see TableRepository#storeRelationships(Table) for an example implementation */ - public abstract void addRelationships(T entity) throws IOException; + public abstract void storeRelationships(T entity) throws IOException; /** * PATCH operations can't overwrite certain fields, such as entity ID, fullyQualifiedNames etc. Instead of throwing * an error, we take lenient approach of ignoring the user error and restore those attributes based on what is - * already stored as original entity. + * already stored in the original entity. */ public abstract void restorePatchAttributes(T original, T updated) throws IOException, ParseException; @@ -376,7 +390,7 @@ public abstract class EntityRepository { private T createNewEntity(T entity) throws IOException { storeEntity(entity, false); - addRelationships(entity); + storeRelationships(entity); return entity; } @@ -393,8 +407,15 @@ public abstract class EntityRepository { } /** - * Class that performs PUT and PATCH update operation. Override {@code entitySpecificUpdate()} to add - * additional entity specific fields to be updated. + * Class that performs PUT and PATCH update operation. It takes an updated entity and original entity. + * Performs comparison between then and updates the stored entity and also updates all the relationships. This class + * also tracks the changes between original and updated to version the entity and produce change events. + * + *

+ * + * Common entity attributes such as description, displayName, owner, tags are handled by this class. + * Override {@code entitySpecificUpdate()} to add additional entity specific fields to be updated. + * @see TableUpdater#entitySpecificUpdate() for example. */ public class EntityUpdater { protected final EntityInterface original; @@ -409,6 +430,9 @@ public abstract class EntityRepository { this.patchOperation = patchOperation; } + /** + * Compare original and updated entities and perform updates. Update the entity version and track changes. + */ public final void update() throws IOException, ParseException { updated.setId(original.getId()); updateDescription(); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java index 37bf4daf9ea..274535c4496 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java @@ -129,7 +129,7 @@ public class IngestionRepository extends EntityRepository { } @Override - public void addRelationships(Ingestion ingestion) throws IOException { + public void storeRelationships(Ingestion ingestion) throws IOException { EntityReference service = ingestion.getService(); dao.relationshipDAO().insert(service.getId().toString(), ingestion.getId().toString(), service.getType(), Entity.INGESTION, Relationship.CONTAINS.ordinal()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java index a61527a7351..64965c23f9d 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java @@ -177,7 +177,7 @@ public class LocationRepository extends EntityRepository { } @Override - public void addRelationships(Location location) throws IOException { + public void storeRelationships(Location location) throws IOException { // Add location owner relationship EntityUtil.setOwner(dao.relationshipDAO(), location.getId(), Entity.LOCATION, location.getOwner()); dao.relationshipDAO().insert(location.getService().getId().toString(), location.getId().toString(), diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java index 91b93e9472d..32c4e0def77 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java @@ -89,7 +89,7 @@ public class MessagingServiceRepository extends EntityRepository { } @Override - public void addRelationships(Metrics metrics) throws IOException { + public void storeRelationships(Metrics metrics) throws IOException { dao.relationshipDAO().insert(metrics.getService().getId().toString(), metrics.getId().toString(), metrics.getService().getType(), Entity.METRICS, Relationship.CONTAINS.ordinal()); setOwner(metrics, metrics.getOwner()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java index 42f79ad5012..0d8ac7156c6 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java @@ -160,7 +160,7 @@ public class MlModelRepository extends EntityRepository { } @Override - public void addRelationships(MlModel mlModel) throws IOException { + public void storeRelationships(MlModel mlModel) throws IOException { EntityUtil.setOwner(dao.relationshipDAO(), mlModel.getId(), Entity.MLMODEL, mlModel.getOwner()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java index c7dac0ba3e6..00b61c8f5b4 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java @@ -143,7 +143,7 @@ public class PipelineRepository extends EntityRepository { } @Override - public void addRelationships(Pipeline pipeline) throws IOException { + public void storeRelationships(Pipeline pipeline) throws IOException { EntityReference service = pipeline.getService(); dao.relationshipDAO().insert(service.getId().toString(), pipeline.getId().toString(), service.getType(), Entity.PIPELINE, Relationship.CONTAINS.ordinal()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java index 080a9ec107b..3e1d47841c4 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java @@ -89,7 +89,7 @@ public class PipelineServiceRepository extends EntityRepository } @Override - public void addRelationships(PipelineService entity) throws IOException { + public void storeRelationships(PipelineService entity) throws IOException { } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java index bc3209a8e5e..1ee3cdbb367 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java @@ -122,7 +122,7 @@ public class PolicyRepository extends EntityRepository { } @Override - public void addRelationships(Policy policy) throws IOException { + public void storeRelationships(Policy policy) throws IOException { // Add policy owner relationship setOwner(policy, policy.getOwner()); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java index 9c8c439004c..21fd5342b7e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java @@ -76,7 +76,7 @@ public class ReportRepository extends EntityRepository { } @Override - public void addRelationships(Report entity) throws IOException { + public void storeRelationships(Report entity) throws IOException { // TODO } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java index 1d2a8f318ac..dfa8a8c74b4 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java @@ -84,7 +84,7 @@ public class StorageServiceRepository extends EntityRepository { } @Override - public void addRelationships(StorageService entity) throws IOException { + public void storeRelationships(StorageService entity) throws IOException { } public static class StorageServiceEntityInterface implements EntityInterface { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java index 8803806ed0b..bcc78981e0f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java @@ -304,7 +304,7 @@ public class TableRepository extends EntityRepository { } @Override - public void addRelationships(Table table) throws IOException { + public void storeRelationships(Table table) throws IOException { // Add relationship from database to table String databaseId = table.getDatabase().getId().toString(); dao.relationshipDAO().insert(databaseId, table.getId().toString(), Entity.DATABASE, Entity.TABLE, diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java index 9595e216b8a..365a940a524 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java @@ -130,7 +130,7 @@ public class TeamRepository extends EntityRepository { } @Override - public void addRelationships(Team team) throws IOException { + public void storeRelationships(Team team) throws IOException { for (EntityReference user : Optional.ofNullable(team.getUsers()).orElse(Collections.emptyList())) { dao.relationshipDAO().insert(team.getId().toString(), user.getId().toString(), "team", "user", Relationship.CONTAINS.ordinal()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java index 8f04ed8a5b2..286bc708463 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java @@ -101,7 +101,7 @@ public class TopicRepository extends EntityRepository { } @Override - public void addRelationships(Topic topic) throws IOException { + public void storeRelationships(Topic topic) throws IOException { setService(topic, topic.getService()); setOwner(topic, topic.getOwner()); applyTags(topic); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java index 00548429315..5a348d5f774 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java @@ -90,7 +90,7 @@ public class UserRepository extends EntityRepository { } @Override - public void addRelationships(User user) throws IOException { + public void storeRelationships(User user) throws IOException { assignTeams(user, user.getTeams()); }