diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java b/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java index f0cd0b835cc..6468c09f7e7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java @@ -82,7 +82,7 @@ public class TypeRegistry { for (CustomProperty property : listOrEmpty(type.getCustomProperties())) { if (TYPES.get(property.getPropertyType().getName()) == null) { throw EntityNotFoundException.byMessage( - CatalogExceptionMessage.entityNotFound(Entity.TYPE, property.getPropertyType().getId())); + CatalogExceptionMessage.entityNotFound(Entity.TYPE, property.getPropertyType().getName())); } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java index 30ee0365e0c..3faba628c23 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java @@ -28,7 +28,7 @@ import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.FullyQualifiedName; public class DatabaseRepository extends EntityRepository { - private static final String DATABASE_UPDATE_FIELDS = "owner,tags"; + private static final String DATABASE_UPDATE_FIELDS = "owner,tags,extension"; private static final String DATABASE_PATCH_FIELDS = DATABASE_UPDATE_FIELDS; public DatabaseRepository(CollectionDAO dao) { @@ -98,9 +98,25 @@ public class DatabaseRepository extends EntityRepository { .withId(original.getId()); } + @Override + public EntityRepository.EntityUpdater getUpdater(Database original, Database updated, Operation operation) { + return new DatabaseUpdater(original, updated, operation); + } + private void populateService(Database database) throws IOException { DatabaseService service = Entity.getEntity(database.getService(), "", Include.NON_DELETED); database.setService(service.getEntityReference()); database.setServiceType(service.getServiceType()); } + + public class DatabaseUpdater extends EntityUpdater { + public DatabaseUpdater(Database original, Database updated, Operation operation) { + super(original, updated, operation); + } + + @Override + public void entitySpecificUpdate() throws IOException { + recordChange("retentionPeriod", original.getRetentionPeriod(), updated.getRetentionPeriod()); + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java index e60b3e1f6d5..032da147cf5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java @@ -19,6 +19,7 @@ import static org.openmetadata.service.Entity.FIELD_OWNER; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.UUID; import org.openmetadata.schema.entity.data.Database; import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.type.EntityReference; @@ -32,7 +33,7 @@ import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.FullyQualifiedName; public class DatabaseSchemaRepository extends EntityRepository { - private static final String DATABASE_SCHEMA_UPDATE_FIELDS = "owner,tags"; + private static final String DATABASE_SCHEMA_UPDATE_FIELDS = "owner,tags,extension"; private static final String DATABASE_SCHEMA_PATCH_FIELDS = DATABASE_SCHEMA_UPDATE_FIELDS; public DatabaseSchemaRepository(CollectionDAO dao) { @@ -102,6 +103,21 @@ public class DatabaseSchemaRepository extends EntityRepository { schema.withDatabase(databaseRef).withService(database.getService()); } + @Override + public void setInheritedFields(DatabaseSchema schema) throws IOException { + Database database = Entity.getEntity(schema.getDatabase(), "owner", Include.ALL); + setInheritedProperties(schema, schema.getDatabase().getId()); + } + + public void setInheritedProperties(DatabaseSchema schema, UUID databaseId) throws IOException { + Database database = null; + // If schema does not have its own retention period, then inherit parent database retention period + if (schema.getRetentionPeriod() == null) { + database = database == null ? Entity.getEntity(Entity.DATABASE, databaseId, "", ALL) : database; + schema.withRetentionPeriod(database.getRetentionPeriod()); + } + } + @Override public void restorePatchAttributes(DatabaseSchema original, DatabaseSchema updated) { // Patch can't make changes to following fields. Ignore the changes @@ -112,6 +128,12 @@ public class DatabaseSchemaRepository extends EntityRepository { .withId(original.getId()); } + @Override + public EntityRepository.EntityUpdater getUpdater( + DatabaseSchema original, DatabaseSchema updated, Operation operation) { + return new DatabaseSchemaUpdater(original, updated, operation); + } + private void populateDatabase(DatabaseSchema schema) throws IOException { Database database = Entity.getEntity(schema.getDatabase(), "owner", ALL); schema @@ -124,4 +146,15 @@ public class DatabaseSchemaRepository extends EntityRepository { schema.withOwner(database.getOwner().withDescription("inherited")); } } + + public class DatabaseSchemaUpdater extends EntityUpdater { + public DatabaseSchemaUpdater(DatabaseSchema original, DatabaseSchema updated, Operation operation) { + super(original, updated, operation); + } + + @Override + public void entitySpecificUpdate() throws IOException { + recordChange("retentionPeriod", original.getRetentionPeriod(), updated.getRetentionPeriod()); + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 0cfcde6faa1..c229063761a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -244,6 +244,16 @@ public abstract class EntityRepository { */ public abstract void storeRelationships(T entity) throws IOException; + /** + * This method is called to set inherited property that an entity inherits from its parent. + * + * @see TableRepository#setInheritedFields(Table) for an example implementation + */ + @SuppressWarnings("unused") + public void setInheritedFields(T entity) throws IOException { + // Override to set inherited properties + } + /** * PATCH operations can't overwrite certain fields, such as entity ID, fullyQualifiedNames etc. Instead of throwing an * error, we take lenient approach of ignoring the user error and restore those attributes based on what is already @@ -518,6 +528,7 @@ public abstract class EntityRepository { entity.setTags(fields.contains(FIELD_TAGS) ? getTags(entity.getFullyQualifiedName()) : null); entity.setExtension(fields.contains(FIELD_EXTENSION) ? getExtension(entity) : null); setFields(entity, fields); + setInheritedFields(entity); return entity; } @@ -574,6 +585,7 @@ public abstract class EntityRepository { EntityUpdater entityUpdater = getUpdater(original, updated, Operation.PUT); entityUpdater.update(); String change = entityUpdater.fieldsChanged() ? RestUtil.ENTITY_UPDATED : RestUtil.ENTITY_NO_CHANGE; + setInheritedFields(updated); return new PutResponse<>(Status.OK, withHref(uriInfo, updated), change); } @@ -595,6 +607,7 @@ public abstract class EntityRepository { EntityUpdater entityUpdater = getUpdater(original, updated, Operation.PATCH); entityUpdater.update(); String change = entityUpdater.fieldsChanged() ? RestUtil.ENTITY_UPDATED : RestUtil.ENTITY_NO_CHANGE; + setInheritedFields(updated); return new PatchResponse<>(Status.OK, withHref(uriInfo, updated), change); } @@ -837,6 +850,7 @@ public abstract class EntityRepository { storeEntity(entity, false); storeExtension(entity); storeRelationships(entity); + setInheritedFields(entity); return entity; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index f12e6720a76..3083e962977 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -136,6 +136,19 @@ public class TableRepository extends EntityRepository { return table; } + @Override + public void setInheritedFields(Table table) throws IOException { + setInheritedProperties(table, table.getDatabaseSchema().getId()); + } + + public void setInheritedProperties(Table table, UUID schemaId) throws IOException { + // If table does not have retention period, then inherit it from parent databaseSchema + if (table.getRetentionPeriod() == null) { + DatabaseSchema schema = Entity.getEntity(DATABASE_SCHEMA, schemaId, "", ALL); + table.withRetentionPeriod(schema.getRetentionPeriod()); + } + } + private void setDefaultFields(Table table) throws IOException { EntityReference schemaRef = getContainer(table.getId()); DatabaseSchema schema = Entity.getEntity(schemaRef, "", ALL); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java index b666a6c0791..a20f4ae19c3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java @@ -86,7 +86,7 @@ public class DatabaseResource extends EntityResource { private Table getTable(CreateTable create, String user) throws IOException { return validateNewTable( - copy(new Table(), create, user) - .withColumns(create.getColumns()) - .withTableConstraints(create.getTableConstraints()) - .withTablePartition(create.getTablePartition()) - .withTableType(create.getTableType()) - .withTags(create.getTags()) - .withViewDefinition(create.getViewDefinition()) - .withTableProfilerConfig(create.getTableProfilerConfig()) - .withDatabaseSchema(getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema()))); + copy(new Table(), create, user) + .withColumns(create.getColumns()) + .withTableConstraints(create.getTableConstraints()) + .withTablePartition(create.getTablePartition()) + .withTableType(create.getTableType()) + .withTags(create.getTags()) + .withViewDefinition(create.getViewDefinition()) + .withTableProfilerConfig(create.getTableProfilerConfig()) + .withDatabaseSchema(getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema()))) + .withRetentionPeriod(create.getRetentionPeriod()); } private CustomMetric getCustomMetric(SecurityContext securityContext, CreateCustomMetric create) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index e723e1d4046..70482a6a6f0 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -253,9 +253,6 @@ public abstract class EntityResourceTest typeResourceTest.patchEntity(id, json1, finalEntity, ADMIN_AUTH_HEADERS), NOT_FOUND, - CatalogExceptionMessage.entityNotFound(Entity.TYPE, invalidType.getId())); + CatalogExceptionMessage.entityNotFound(Entity.TYPE, invalidType.getName())); // Now POST an entity with extension that includes custom field intA ObjectMapper mapper = new ObjectMapper(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index f7d77911748..78c8b78b1b9 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -1724,6 +1724,33 @@ public class TableResourceTest extends EntityResourceTest { assertReference(USER2_REF, schema.getOwner()); // Owner remains the same } + @Test + void test_retentionPeriod(TestInfo test) throws HttpResponseException { + DatabaseResourceTest databaseTest = new DatabaseResourceTest(); + CreateDatabase createDatabase = databaseTest.createRequest(getEntityName(test)).withRetentionPeriod("P30D"); + Database database = databaseTest.createEntity(createDatabase, ADMIN_AUTH_HEADERS); + assertEquals("P30D", database.getRetentionPeriod()); + + // Ensure database schema retention period is carried over from the parent database + DatabaseSchemaResourceTest schemaResourceTest = new DatabaseSchemaResourceTest(); + CreateDatabaseSchema createDatabaseSchema = + schemaResourceTest.createRequest(test).withDatabase(database.getFullyQualifiedName()); + DatabaseSchema schema = + schemaResourceTest + .createEntity(createDatabaseSchema, ADMIN_AUTH_HEADERS) + .withDatabase(database.getEntityReference()); + assertEquals("P30D", schema.getRetentionPeriod()); + schema = schemaResourceTest.getEntity(schema.getId(), "", ADMIN_AUTH_HEADERS); + assertEquals("P30D", schema.getRetentionPeriod()); + + // Ensure table retention period is carried over from the parent database schema + CreateTable createTable = createRequest(test).withDatabaseSchema(schema.getFullyQualifiedName()); + Table table = createEntity(createTable, ADMIN_AUTH_HEADERS).withDatabase(database.getEntityReference()); + assertEquals("P30D", table.getRetentionPeriod()); + table = getEntity(table.getId(), "", ADMIN_AUTH_HEADERS); + assertEquals("P30D", table.getRetentionPeriod()); + } + void assertFields(List
tableList, String fieldsParam) { tableList.forEach(t -> assertFields(t, fieldsParam)); } diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json index dc9de7acf34..b0c7156e89a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json @@ -40,6 +40,14 @@ "description": "Some databases don't support a database/catalog in the hierarchy and use default database. For example, `MySql`. For such databases, set this flag to true to indicate that this is a default database.", "type": "boolean", "default": false + }, + "retentionPeriod" : { + "description": "Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`.", + "$ref": "../../type/basic.json#/definitions/duration" + }, + "extension": { + "description": "Entity extension data with custom attributes added to the entity.", + "$ref": "../../type/basic.json#/definitions/entityExtension" } }, "required": ["name", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json index 688816382d4..d5d3458912c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json @@ -36,6 +36,14 @@ "$ref": "../../type/tagLabel.json" }, "default": null + }, + "retentionPeriod" : { + "description": "Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`.", + "$ref": "../../type/basic.json#/definitions/duration" + }, + "extension": { + "description": "Entity extension data with custom attributes added to the entity.", + "$ref": "../../type/basic.json#/definitions/entityExtension" } }, "required": [ diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json index 36e256e1a29..c7b477da26c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json @@ -66,6 +66,10 @@ "$ref": "../../type/basic.json#/definitions/sqlQuery", "default": null }, + "retentionPeriod" : { + "description": "Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`.", + "$ref": "../../type/basic.json#/definitions/duration" + }, "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/database.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/database.json index 30929acdfe4..ae951b42ebd 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/database.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/database.json @@ -2,6 +2,7 @@ "$id": "https://open-metadata.org/schema/entity/data/database.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "Database", + "$comment": "@om-entity-type", "description": "This schema defines the Database entity. A database also referred to as Database Catalog is a collection of schemas.", "type": "object", "javaType": "org.openmetadata.schema.entity.data.Database", @@ -98,6 +99,14 @@ "description": "When `true` indicates the entity has been soft deleted.", "type": "boolean", "default": false + }, + "retentionPeriod" : { + "description": "Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`.", + "$ref": "../../type/basic.json#/definitions/duration" + }, + "extension": { + "description": "Entity extension data with custom attributes added to the entity.", + "$ref": "../../type/basic.json#/definitions/entityExtension" } }, "required": ["name", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json index f5c9be5650e..38b8e3a4baf 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json @@ -2,6 +2,7 @@ "$id": "https://open-metadata.org/schema/entity/data/databaseSchema.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "Database Schema", + "$comment": "@om-entity-type", "description": "This schema defines the Database Schema entity. A `Database Schema` is collection of tables, views, stored procedures, and other database objects.", "type": "object", "javaType": "org.openmetadata.schema.entity.data.DatabaseSchema", @@ -93,6 +94,14 @@ "description": "When `true` indicates the entity has been soft deleted.", "type": "boolean", "default": false + }, + "retentionPeriod" : { + "description": "Retention period of the data in the database schema. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`. When not set, the retention period is inherited from the parent database, if it exists.", + "$ref": "../../type/basic.json#/definitions/duration" + }, + "extension": { + "description": "Entity extension data with custom attributes added to the entity.", + "$ref": "../../type/basic.json#/definitions/entityExtension" } }, "required": ["name", "database", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json index 903f97e8e58..cfc4935fc97 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json @@ -982,6 +982,10 @@ "type": "boolean", "default": false }, + "retentionPeriod" : { + "description": "Retention period of the data in the table. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`. When not set, the retention period is inherited from the parent database schema, if it exists.", + "$ref": "../../type/basic.json#/definitions/duration" + }, "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension"