11287 Task - Add retention period and customer properties to Database, Schema, and Tables (#11511)

openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java

@@ -82,7 +82,7 @@ public class TypeRegistry {
     for (CustomProperty property : listOrEmpty(type.getCustomProperties())) {
       if (TYPES.get(property.getPropertyType().getName()) == null) {
         throw EntityNotFoundException.byMessage(
-            CatalogExceptionMessage.entityNotFound(Entity.TYPE, property.getPropertyType().getId()));
+            CatalogExceptionMessage.entityNotFound(Entity.TYPE, property.getPropertyType().getName()));
       }
     }
   }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java

@@ -28,7 +28,7 @@ import org.openmetadata.service.util.EntityUtil.Fields;
 import org.openmetadata.service.util.FullyQualifiedName;
 
 public class DatabaseRepository extends EntityRepository<Database> {
-  private static final String DATABASE_UPDATE_FIELDS = "owner,tags";
+  private static final String DATABASE_UPDATE_FIELDS = "owner,tags,extension";
   private static final String DATABASE_PATCH_FIELDS = DATABASE_UPDATE_FIELDS;
 
   public DatabaseRepository(CollectionDAO dao) {
@@ -98,9 +98,25 @@ public class DatabaseRepository extends EntityRepository<Database> {
         .withId(original.getId());
   }
 
+  @Override
+  public EntityRepository<Database>.EntityUpdater getUpdater(Database original, Database updated, Operation operation) {
+    return new DatabaseUpdater(original, updated, operation);
+  }
+
   private void populateService(Database database) throws IOException {
     DatabaseService service = Entity.getEntity(database.getService(), "", Include.NON_DELETED);
     database.setService(service.getEntityReference());
     database.setServiceType(service.getServiceType());
   }
+
+  public class DatabaseUpdater extends EntityUpdater {
+    public DatabaseUpdater(Database original, Database updated, Operation operation) {
+      super(original, updated, operation);
+    }
+
+    @Override
+    public void entitySpecificUpdate() throws IOException {
+      recordChange("retentionPeriod", original.getRetentionPeriod(), updated.getRetentionPeriod());
+    }
+  }
 }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java

@@ -19,6 +19,7 @@ import static org.openmetadata.service.Entity.FIELD_OWNER;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.UUID;
 import org.openmetadata.schema.entity.data.Database;
 import org.openmetadata.schema.entity.data.DatabaseSchema;
 import org.openmetadata.schema.type.EntityReference;
@@ -32,7 +33,7 @@ import org.openmetadata.service.util.EntityUtil.Fields;
 import org.openmetadata.service.util.FullyQualifiedName;
 
 public class DatabaseSchemaRepository extends EntityRepository<DatabaseSchema> {
-  private static final String DATABASE_SCHEMA_UPDATE_FIELDS = "owner,tags";
+  private static final String DATABASE_SCHEMA_UPDATE_FIELDS = "owner,tags,extension";
   private static final String DATABASE_SCHEMA_PATCH_FIELDS = DATABASE_SCHEMA_UPDATE_FIELDS;
 
   public DatabaseSchemaRepository(CollectionDAO dao) {
@@ -102,6 +103,21 @@ public class DatabaseSchemaRepository extends EntityRepository<DatabaseSchema> {
     schema.withDatabase(databaseRef).withService(database.getService());
   }
 
+  @Override
+  public void setInheritedFields(DatabaseSchema schema) throws IOException {
+    Database database = Entity.getEntity(schema.getDatabase(), "owner", Include.ALL);
+    setInheritedProperties(schema, schema.getDatabase().getId());
+  }
+
+  public void setInheritedProperties(DatabaseSchema schema, UUID databaseId) throws IOException {
+    Database database = null;
+    // If schema does not have its own retention period, then inherit parent database retention period
+    if (schema.getRetentionPeriod() == null) {
+      database = database == null ? Entity.getEntity(Entity.DATABASE, databaseId, "", ALL) : database;
+      schema.withRetentionPeriod(database.getRetentionPeriod());
+    }
+  }
+
   @Override
   public void restorePatchAttributes(DatabaseSchema original, DatabaseSchema updated) {
     // Patch can't make changes to following fields. Ignore the changes
@@ -112,6 +128,12 @@ public class DatabaseSchemaRepository extends EntityRepository<DatabaseSchema> {
         .withId(original.getId());
   }
 
+  @Override
+  public EntityRepository<DatabaseSchema>.EntityUpdater getUpdater(
+      DatabaseSchema original, DatabaseSchema updated, Operation operation) {
+    return new DatabaseSchemaUpdater(original, updated, operation);
+  }
+
   private void populateDatabase(DatabaseSchema schema) throws IOException {
     Database database = Entity.getEntity(schema.getDatabase(), "owner", ALL);
     schema
@@ -124,4 +146,15 @@ public class DatabaseSchemaRepository extends EntityRepository<DatabaseSchema> {
       schema.withOwner(database.getOwner().withDescription("inherited"));
     }
   }
+
+  public class DatabaseSchemaUpdater extends EntityUpdater {
+    public DatabaseSchemaUpdater(DatabaseSchema original, DatabaseSchema updated, Operation operation) {
+      super(original, updated, operation);
+    }
+
+    @Override
+    public void entitySpecificUpdate() throws IOException {
+      recordChange("retentionPeriod", original.getRetentionPeriod(), updated.getRetentionPeriod());
+    }
+  }
 }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java

@@ -244,6 +244,16 @@ public abstract class EntityRepository<T extends EntityInterface> {
    */
   public abstract void storeRelationships(T entity) throws IOException;
 
+  /**
+   * This method is called to set inherited property that an entity inherits from its parent.
+   *
+   * @see TableRepository#setInheritedFields(Table) for an example implementation
+   */
+  @SuppressWarnings("unused")
+  public void setInheritedFields(T entity) throws IOException {
+    // Override to set inherited properties
+  }
+
   /**
    * PATCH operations can't overwrite certain fields, such as entity ID, fullyQualifiedNames etc. Instead of throwing an
    * error, we take lenient approach of ignoring the user error and restore those attributes based on what is already
@@ -518,6 +528,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
     entity.setTags(fields.contains(FIELD_TAGS) ? getTags(entity.getFullyQualifiedName()) : null);
     entity.setExtension(fields.contains(FIELD_EXTENSION) ? getExtension(entity) : null);
     setFields(entity, fields);
+    setInheritedFields(entity);
     return entity;
   }
 
@@ -574,6 +585,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
     EntityUpdater entityUpdater = getUpdater(original, updated, Operation.PUT);
     entityUpdater.update();
     String change = entityUpdater.fieldsChanged() ? RestUtil.ENTITY_UPDATED : RestUtil.ENTITY_NO_CHANGE;
+    setInheritedFields(updated);
     return new PutResponse<>(Status.OK, withHref(uriInfo, updated), change);
   }
 
@@ -595,6 +607,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
     EntityUpdater entityUpdater = getUpdater(original, updated, Operation.PATCH);
     entityUpdater.update();
     String change = entityUpdater.fieldsChanged() ? RestUtil.ENTITY_UPDATED : RestUtil.ENTITY_NO_CHANGE;
+    setInheritedFields(updated);
     return new PatchResponse<>(Status.OK, withHref(uriInfo, updated), change);
   }
 
@@ -837,6 +850,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
     storeEntity(entity, false);
     storeExtension(entity);
     storeRelationships(entity);
+    setInheritedFields(entity);
     return entity;
   }
 

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java

@@ -136,6 +136,19 @@ public class TableRepository extends EntityRepository<Table> {
     return table;
   }
 
+  @Override
+  public void setInheritedFields(Table table) throws IOException {
+    setInheritedProperties(table, table.getDatabaseSchema().getId());
+  }
+
+  public void setInheritedProperties(Table table, UUID schemaId) throws IOException {
+    // If table does not have retention period, then inherit it from parent databaseSchema
+    if (table.getRetentionPeriod() == null) {
+      DatabaseSchema schema = Entity.getEntity(DATABASE_SCHEMA, schemaId, "", ALL);
+      table.withRetentionPeriod(schema.getRetentionPeriod());
+    }
+  }
+
   private void setDefaultFields(Table table) throws IOException {
     EntityReference schemaRef = getContainer(table.getId());
     DatabaseSchema schema = Entity.getEntity(schemaRef, "", ALL);

openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java

@@ -86,7 +86,7 @@ public class DatabaseResource extends EntityResource<Database, DatabaseRepositor
     DatabaseList() {}
   }
 
-  static final String FIELDS = "owner,databaseSchemas,usageSummary,location,tags";
+  static final String FIELDS = "owner,databaseSchemas,usageSummary,location,tags,extension";
 
   @GET
   @Operation(
@@ -384,6 +384,7 @@ public class DatabaseResource extends EntityResource<Database, DatabaseRepositor
 
   private Database getDatabase(CreateDatabase create, String user) throws IOException {
     return copy(new Database(), create, user)
-        .withService(getEntityReference(Entity.DATABASE_SERVICE, create.getService()));
+        .withService(getEntityReference(Entity.DATABASE_SERVICE, create.getService()))
+        .withRetentionPeriod(create.getRetentionPeriod());
   }
 }

openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java

@@ -86,7 +86,7 @@ public class DatabaseSchemaResource extends EntityResource<DatabaseSchema, Datab
     DatabaseSchemaList() {}
   }
 
-  static final String FIELDS = "owner,tables,usageSummary,tags";
+  static final String FIELDS = "owner,tables,usageSummary,tags,extension";
 
   @GET
   @Operation(
@@ -388,6 +388,7 @@ public class DatabaseSchemaResource extends EntityResource<DatabaseSchema, Datab
   private DatabaseSchema getDatabaseSchema(CreateDatabaseSchema create, String user) throws IOException {
     return copy(new DatabaseSchema(), create, user)
         .withDatabase(getEntityReference(Entity.DATABASE, create.getDatabase()))
-        .withTags(create.getTags());
+        .withTags(create.getTags())
+        .withRetentionPeriod(create.getRetentionPeriod());
   }
 }

openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java

@@ -907,15 +907,16 @@ public class TableResource extends EntityResource<Table, TableRepository> {
 
   private Table getTable(CreateTable create, String user) throws IOException {
     return validateNewTable(
-        copy(new Table(), create, user)
+            copy(new Table(), create, user)
-            .withColumns(create.getColumns())
+                .withColumns(create.getColumns())
-            .withTableConstraints(create.getTableConstraints())
+                .withTableConstraints(create.getTableConstraints())
-            .withTablePartition(create.getTablePartition())
+                .withTablePartition(create.getTablePartition())
-            .withTableType(create.getTableType())
+                .withTableType(create.getTableType())
-            .withTags(create.getTags())
+                .withTags(create.getTags())
-            .withViewDefinition(create.getViewDefinition())
+                .withViewDefinition(create.getViewDefinition())
-            .withTableProfilerConfig(create.getTableProfilerConfig())
+                .withTableProfilerConfig(create.getTableProfilerConfig())
-            .withDatabaseSchema(getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema())));
+                .withDatabaseSchema(getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema())))
+        .withRetentionPeriod(create.getRetentionPeriod());
   }
 
   private CustomMetric getCustomMetric(SecurityContext securityContext, CreateCustomMetric create) {

openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java

@@ -253,9 +253,6 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
 
   public static EntityReference S3_OBJECT_STORE_SERVICE_REFERENCE;
 
-  public static EntityReference AWS_STORAGE_SERVICE_REFERENCE;
-  public static EntityReference GCP_STORAGE_SERVICE_REFERENCE;
-
   public static EntityReference AMUNDSEN_SERVICE_REFERENCE;
   public static EntityReference ATLAS_SERVICE_REFERENCE;
 
@@ -1470,7 +1467,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
     assertResponse(
         () -> typeResourceTest.patchEntity(id, json1, finalEntity, ADMIN_AUTH_HEADERS),
         NOT_FOUND,
-        CatalogExceptionMessage.entityNotFound(Entity.TYPE, invalidType.getId()));
+        CatalogExceptionMessage.entityNotFound(Entity.TYPE, invalidType.getName()));
 
     // Now POST an entity with extension that includes custom field intA
     ObjectMapper mapper = new ObjectMapper();

openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java

@@ -1724,6 +1724,33 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
     assertReference(USER2_REF, schema.getOwner()); // Owner remains the same
   }
 
+  @Test
+  void test_retentionPeriod(TestInfo test) throws HttpResponseException {
+    DatabaseResourceTest databaseTest = new DatabaseResourceTest();
+    CreateDatabase createDatabase = databaseTest.createRequest(getEntityName(test)).withRetentionPeriod("P30D");
+    Database database = databaseTest.createEntity(createDatabase, ADMIN_AUTH_HEADERS);
+    assertEquals("P30D", database.getRetentionPeriod());
+
+    // Ensure database schema retention period is carried over from the parent database
+    DatabaseSchemaResourceTest schemaResourceTest = new DatabaseSchemaResourceTest();
+    CreateDatabaseSchema createDatabaseSchema =
+        schemaResourceTest.createRequest(test).withDatabase(database.getFullyQualifiedName());
+    DatabaseSchema schema =
+        schemaResourceTest
+            .createEntity(createDatabaseSchema, ADMIN_AUTH_HEADERS)
+            .withDatabase(database.getEntityReference());
+    assertEquals("P30D", schema.getRetentionPeriod());
+    schema = schemaResourceTest.getEntity(schema.getId(), "", ADMIN_AUTH_HEADERS);
+    assertEquals("P30D", schema.getRetentionPeriod());
+
+    // Ensure table retention period is carried over from the parent database schema
+    CreateTable createTable = createRequest(test).withDatabaseSchema(schema.getFullyQualifiedName());
+    Table table = createEntity(createTable, ADMIN_AUTH_HEADERS).withDatabase(database.getEntityReference());
+    assertEquals("P30D", table.getRetentionPeriod());
+    table = getEntity(table.getId(), "", ADMIN_AUTH_HEADERS);
+    assertEquals("P30D", table.getRetentionPeriod());
+  }
+
   void assertFields(List<Table> tableList, String fieldsParam) {
     tableList.forEach(t -> assertFields(t, fieldsParam));
   }

openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json

@@ -40,6 +40,14 @@
       "description": "Some databases don't support a database/catalog in the hierarchy and use default database. For example, `MySql`. For such databases, set this flag to true to indicate that this is a default database.",
       "type": "boolean",
       "default": false
+    },
+    "retentionPeriod" : {
+      "description": "Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`.",
+      "$ref": "../../type/basic.json#/definitions/duration"
+    },
+    "extension": {
+      "description": "Entity extension data with custom attributes added to the entity.",
+      "$ref": "../../type/basic.json#/definitions/entityExtension"
     }
   },
   "required": ["name", "service"],

openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json

@@ -36,6 +36,14 @@
         "$ref": "../../type/tagLabel.json"
       },
       "default": null
+    },
+    "retentionPeriod" : {
+      "description": "Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`.",
+      "$ref": "../../type/basic.json#/definitions/duration"
+    },
+    "extension": {
+      "description": "Entity extension data with custom attributes added to the entity.",
+      "$ref": "../../type/basic.json#/definitions/entityExtension"
     }
   },
   "required": [

openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json

@@ -66,6 +66,10 @@
       "$ref": "../../type/basic.json#/definitions/sqlQuery",
       "default": null
     },
+    "retentionPeriod" : {
+      "description": "Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`.",
+      "$ref": "../../type/basic.json#/definitions/duration"
+    },
     "extension": {
       "description": "Entity extension data with custom attributes added to the entity.",
       "$ref": "../../type/basic.json#/definitions/entityExtension"

openmetadata-spec/src/main/resources/json/schema/entity/data/database.json

@@ -2,6 +2,7 @@
   "$id": "https://open-metadata.org/schema/entity/data/database.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Database",
+  "$comment": "@om-entity-type",
   "description": "This schema defines the Database entity. A database also referred to as Database Catalog is a collection of schemas.",
   "type": "object",
   "javaType": "org.openmetadata.schema.entity.data.Database",
@@ -98,6 +99,14 @@
       "description": "When `true` indicates the entity has been soft deleted.",
       "type": "boolean",
       "default": false
+    },
+    "retentionPeriod" : {
+      "description": "Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`.",
+      "$ref": "../../type/basic.json#/definitions/duration"
+    },
+    "extension": {
+      "description": "Entity extension data with custom attributes added to the entity.",
+      "$ref": "../../type/basic.json#/definitions/entityExtension"
     }
   },
   "required": ["name", "service"],

openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json

@@ -2,6 +2,7 @@
   "$id": "https://open-metadata.org/schema/entity/data/databaseSchema.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Database Schema",
+  "$comment": "@om-entity-type",
   "description": "This schema defines the Database Schema entity. A `Database Schema` is collection of tables, views, stored procedures, and other database objects.",
   "type": "object",
   "javaType": "org.openmetadata.schema.entity.data.DatabaseSchema",
@@ -93,6 +94,14 @@
       "description": "When `true` indicates the entity has been soft deleted.",
       "type": "boolean",
       "default": false
+    },
+    "retentionPeriod" : {
+      "description": "Retention period of the data in the database schema. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`. When not set, the retention period is inherited from the parent database, if it exists.",
+      "$ref": "../../type/basic.json#/definitions/duration"
+    },
+    "extension": {
+      "description": "Entity extension data with custom attributes added to the entity.",
+      "$ref": "../../type/basic.json#/definitions/entityExtension"
     }
   },
   "required": ["name", "database", "service"],

openmetadata-spec/src/main/resources/json/schema/entity/data/table.json

@@ -982,6 +982,10 @@
       "type": "boolean",
       "default": false
     },
+    "retentionPeriod" : {
+      "description": "Retention period of the data in the table. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`. When not set, the retention period is inherited from the parent database schema, if it exists.",
+      "$ref": "../../type/basic.json#/definitions/duration"
+    },
     "extension": {
       "description": "Entity extension data with custom attributes added to the entity.",
       "$ref": "../../type/basic.json#/definitions/entityExtension"