11287 Task - Add retention period and customer properties to Database, Schema, and Tables (#11511)

This commit is contained in:
Suresh Srinivas 2023-05-10 16:08:21 -07:00 committed by GitHub
parent 944a07cbd5
commit 14b622efd0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 165 additions and 20 deletions

View File

@ -82,7 +82,7 @@ public class TypeRegistry {
for (CustomProperty property : listOrEmpty(type.getCustomProperties())) {
if (TYPES.get(property.getPropertyType().getName()) == null) {
throw EntityNotFoundException.byMessage(
CatalogExceptionMessage.entityNotFound(Entity.TYPE, property.getPropertyType().getId()));
CatalogExceptionMessage.entityNotFound(Entity.TYPE, property.getPropertyType().getName()));
}
}
}

View File

@ -28,7 +28,7 @@ import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.FullyQualifiedName;
public class DatabaseRepository extends EntityRepository<Database> {
private static final String DATABASE_UPDATE_FIELDS = "owner,tags";
private static final String DATABASE_UPDATE_FIELDS = "owner,tags,extension";
private static final String DATABASE_PATCH_FIELDS = DATABASE_UPDATE_FIELDS;
public DatabaseRepository(CollectionDAO dao) {
@ -98,9 +98,25 @@ public class DatabaseRepository extends EntityRepository<Database> {
.withId(original.getId());
}
@Override
public EntityRepository<Database>.EntityUpdater getUpdater(Database original, Database updated, Operation operation) {
return new DatabaseUpdater(original, updated, operation);
}
private void populateService(Database database) throws IOException {
DatabaseService service = Entity.getEntity(database.getService(), "", Include.NON_DELETED);
database.setService(service.getEntityReference());
database.setServiceType(service.getServiceType());
}
public class DatabaseUpdater extends EntityUpdater {
public DatabaseUpdater(Database original, Database updated, Operation operation) {
super(original, updated, operation);
}
@Override
public void entitySpecificUpdate() throws IOException {
recordChange("retentionPeriod", original.getRetentionPeriod(), updated.getRetentionPeriod());
}
}
}

View File

@ -19,6 +19,7 @@ import static org.openmetadata.service.Entity.FIELD_OWNER;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import org.openmetadata.schema.entity.data.Database;
import org.openmetadata.schema.entity.data.DatabaseSchema;
import org.openmetadata.schema.type.EntityReference;
@ -32,7 +33,7 @@ import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.FullyQualifiedName;
public class DatabaseSchemaRepository extends EntityRepository<DatabaseSchema> {
private static final String DATABASE_SCHEMA_UPDATE_FIELDS = "owner,tags";
private static final String DATABASE_SCHEMA_UPDATE_FIELDS = "owner,tags,extension";
private static final String DATABASE_SCHEMA_PATCH_FIELDS = DATABASE_SCHEMA_UPDATE_FIELDS;
public DatabaseSchemaRepository(CollectionDAO dao) {
@ -102,6 +103,21 @@ public class DatabaseSchemaRepository extends EntityRepository<DatabaseSchema> {
schema.withDatabase(databaseRef).withService(database.getService());
}
@Override
public void setInheritedFields(DatabaseSchema schema) throws IOException {
Database database = Entity.getEntity(schema.getDatabase(), "owner", Include.ALL);
setInheritedProperties(schema, schema.getDatabase().getId());
}
public void setInheritedProperties(DatabaseSchema schema, UUID databaseId) throws IOException {
Database database = null;
// If schema does not have its own retention period, then inherit parent database retention period
if (schema.getRetentionPeriod() == null) {
database = database == null ? Entity.getEntity(Entity.DATABASE, databaseId, "", ALL) : database;
schema.withRetentionPeriod(database.getRetentionPeriod());
}
}
@Override
public void restorePatchAttributes(DatabaseSchema original, DatabaseSchema updated) {
// Patch can't make changes to following fields. Ignore the changes
@ -112,6 +128,12 @@ public class DatabaseSchemaRepository extends EntityRepository<DatabaseSchema> {
.withId(original.getId());
}
@Override
public EntityRepository<DatabaseSchema>.EntityUpdater getUpdater(
DatabaseSchema original, DatabaseSchema updated, Operation operation) {
return new DatabaseSchemaUpdater(original, updated, operation);
}
private void populateDatabase(DatabaseSchema schema) throws IOException {
Database database = Entity.getEntity(schema.getDatabase(), "owner", ALL);
schema
@ -124,4 +146,15 @@ public class DatabaseSchemaRepository extends EntityRepository<DatabaseSchema> {
schema.withOwner(database.getOwner().withDescription("inherited"));
}
}
public class DatabaseSchemaUpdater extends EntityUpdater {
public DatabaseSchemaUpdater(DatabaseSchema original, DatabaseSchema updated, Operation operation) {
super(original, updated, operation);
}
@Override
public void entitySpecificUpdate() throws IOException {
recordChange("retentionPeriod", original.getRetentionPeriod(), updated.getRetentionPeriod());
}
}
}

View File

@ -244,6 +244,16 @@ public abstract class EntityRepository<T extends EntityInterface> {
*/
public abstract void storeRelationships(T entity) throws IOException;
/**
* This method is called to set inherited property that an entity inherits from its parent.
*
* @see TableRepository#setInheritedFields(Table) for an example implementation
*/
@SuppressWarnings("unused")
public void setInheritedFields(T entity) throws IOException {
// Override to set inherited properties
}
/**
* PATCH operations can't overwrite certain fields, such as entity ID, fullyQualifiedNames etc. Instead of throwing an
* error, we take lenient approach of ignoring the user error and restore those attributes based on what is already
@ -518,6 +528,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
entity.setTags(fields.contains(FIELD_TAGS) ? getTags(entity.getFullyQualifiedName()) : null);
entity.setExtension(fields.contains(FIELD_EXTENSION) ? getExtension(entity) : null);
setFields(entity, fields);
setInheritedFields(entity);
return entity;
}
@ -574,6 +585,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
EntityUpdater entityUpdater = getUpdater(original, updated, Operation.PUT);
entityUpdater.update();
String change = entityUpdater.fieldsChanged() ? RestUtil.ENTITY_UPDATED : RestUtil.ENTITY_NO_CHANGE;
setInheritedFields(updated);
return new PutResponse<>(Status.OK, withHref(uriInfo, updated), change);
}
@ -595,6 +607,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
EntityUpdater entityUpdater = getUpdater(original, updated, Operation.PATCH);
entityUpdater.update();
String change = entityUpdater.fieldsChanged() ? RestUtil.ENTITY_UPDATED : RestUtil.ENTITY_NO_CHANGE;
setInheritedFields(updated);
return new PatchResponse<>(Status.OK, withHref(uriInfo, updated), change);
}
@ -837,6 +850,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
storeEntity(entity, false);
storeExtension(entity);
storeRelationships(entity);
setInheritedFields(entity);
return entity;
}

View File

@ -136,6 +136,19 @@ public class TableRepository extends EntityRepository<Table> {
return table;
}
@Override
public void setInheritedFields(Table table) throws IOException {
setInheritedProperties(table, table.getDatabaseSchema().getId());
}
public void setInheritedProperties(Table table, UUID schemaId) throws IOException {
// If table does not have retention period, then inherit it from parent databaseSchema
if (table.getRetentionPeriod() == null) {
DatabaseSchema schema = Entity.getEntity(DATABASE_SCHEMA, schemaId, "", ALL);
table.withRetentionPeriod(schema.getRetentionPeriod());
}
}
private void setDefaultFields(Table table) throws IOException {
EntityReference schemaRef = getContainer(table.getId());
DatabaseSchema schema = Entity.getEntity(schemaRef, "", ALL);

View File

@ -86,7 +86,7 @@ public class DatabaseResource extends EntityResource<Database, DatabaseRepositor
DatabaseList() {}
}
static final String FIELDS = "owner,databaseSchemas,usageSummary,location,tags";
static final String FIELDS = "owner,databaseSchemas,usageSummary,location,tags,extension";
@GET
@Operation(
@ -384,6 +384,7 @@ public class DatabaseResource extends EntityResource<Database, DatabaseRepositor
private Database getDatabase(CreateDatabase create, String user) throws IOException {
return copy(new Database(), create, user)
.withService(getEntityReference(Entity.DATABASE_SERVICE, create.getService()));
.withService(getEntityReference(Entity.DATABASE_SERVICE, create.getService()))
.withRetentionPeriod(create.getRetentionPeriod());
}
}

View File

@ -86,7 +86,7 @@ public class DatabaseSchemaResource extends EntityResource<DatabaseSchema, Datab
DatabaseSchemaList() {}
}
static final String FIELDS = "owner,tables,usageSummary,tags";
static final String FIELDS = "owner,tables,usageSummary,tags,extension";
@GET
@Operation(
@ -388,6 +388,7 @@ public class DatabaseSchemaResource extends EntityResource<DatabaseSchema, Datab
private DatabaseSchema getDatabaseSchema(CreateDatabaseSchema create, String user) throws IOException {
return copy(new DatabaseSchema(), create, user)
.withDatabase(getEntityReference(Entity.DATABASE, create.getDatabase()))
.withTags(create.getTags());
.withTags(create.getTags())
.withRetentionPeriod(create.getRetentionPeriod());
}
}

View File

@ -915,7 +915,8 @@ public class TableResource extends EntityResource<Table, TableRepository> {
.withTags(create.getTags())
.withViewDefinition(create.getViewDefinition())
.withTableProfilerConfig(create.getTableProfilerConfig())
.withDatabaseSchema(getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema())));
.withDatabaseSchema(getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema())))
.withRetentionPeriod(create.getRetentionPeriod());
}
private CustomMetric getCustomMetric(SecurityContext securityContext, CreateCustomMetric create) {

View File

@ -253,9 +253,6 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
public static EntityReference S3_OBJECT_STORE_SERVICE_REFERENCE;
public static EntityReference AWS_STORAGE_SERVICE_REFERENCE;
public static EntityReference GCP_STORAGE_SERVICE_REFERENCE;
public static EntityReference AMUNDSEN_SERVICE_REFERENCE;
public static EntityReference ATLAS_SERVICE_REFERENCE;
@ -1470,7 +1467,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
assertResponse(
() -> typeResourceTest.patchEntity(id, json1, finalEntity, ADMIN_AUTH_HEADERS),
NOT_FOUND,
CatalogExceptionMessage.entityNotFound(Entity.TYPE, invalidType.getId()));
CatalogExceptionMessage.entityNotFound(Entity.TYPE, invalidType.getName()));
// Now POST an entity with extension that includes custom field intA
ObjectMapper mapper = new ObjectMapper();

View File

@ -1724,6 +1724,33 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
assertReference(USER2_REF, schema.getOwner()); // Owner remains the same
}
@Test
void test_retentionPeriod(TestInfo test) throws HttpResponseException {
DatabaseResourceTest databaseTest = new DatabaseResourceTest();
CreateDatabase createDatabase = databaseTest.createRequest(getEntityName(test)).withRetentionPeriod("P30D");
Database database = databaseTest.createEntity(createDatabase, ADMIN_AUTH_HEADERS);
assertEquals("P30D", database.getRetentionPeriod());
// Ensure database schema retention period is carried over from the parent database
DatabaseSchemaResourceTest schemaResourceTest = new DatabaseSchemaResourceTest();
CreateDatabaseSchema createDatabaseSchema =
schemaResourceTest.createRequest(test).withDatabase(database.getFullyQualifiedName());
DatabaseSchema schema =
schemaResourceTest
.createEntity(createDatabaseSchema, ADMIN_AUTH_HEADERS)
.withDatabase(database.getEntityReference());
assertEquals("P30D", schema.getRetentionPeriod());
schema = schemaResourceTest.getEntity(schema.getId(), "", ADMIN_AUTH_HEADERS);
assertEquals("P30D", schema.getRetentionPeriod());
// Ensure table retention period is carried over from the parent database schema
CreateTable createTable = createRequest(test).withDatabaseSchema(schema.getFullyQualifiedName());
Table table = createEntity(createTable, ADMIN_AUTH_HEADERS).withDatabase(database.getEntityReference());
assertEquals("P30D", table.getRetentionPeriod());
table = getEntity(table.getId(), "", ADMIN_AUTH_HEADERS);
assertEquals("P30D", table.getRetentionPeriod());
}
void assertFields(List<Table> tableList, String fieldsParam) {
tableList.forEach(t -> assertFields(t, fieldsParam));
}

View File

@ -40,6 +40,14 @@
"description": "Some databases don't support a database/catalog in the hierarchy and use default database. For example, `MySql`. For such databases, set this flag to true to indicate that this is a default database.",
"type": "boolean",
"default": false
},
"retentionPeriod" : {
"description": "Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`.",
"$ref": "../../type/basic.json#/definitions/duration"
},
"extension": {
"description": "Entity extension data with custom attributes added to the entity.",
"$ref": "../../type/basic.json#/definitions/entityExtension"
}
},
"required": ["name", "service"],

View File

@ -36,6 +36,14 @@
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"retentionPeriod" : {
"description": "Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`.",
"$ref": "../../type/basic.json#/definitions/duration"
},
"extension": {
"description": "Entity extension data with custom attributes added to the entity.",
"$ref": "../../type/basic.json#/definitions/entityExtension"
}
},
"required": [

View File

@ -66,6 +66,10 @@
"$ref": "../../type/basic.json#/definitions/sqlQuery",
"default": null
},
"retentionPeriod" : {
"description": "Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`.",
"$ref": "../../type/basic.json#/definitions/duration"
},
"extension": {
"description": "Entity extension data with custom attributes added to the entity.",
"$ref": "../../type/basic.json#/definitions/entityExtension"

View File

@ -2,6 +2,7 @@
"$id": "https://open-metadata.org/schema/entity/data/database.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Database",
"$comment": "@om-entity-type",
"description": "This schema defines the Database entity. A database also referred to as Database Catalog is a collection of schemas.",
"type": "object",
"javaType": "org.openmetadata.schema.entity.data.Database",
@ -98,6 +99,14 @@
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
},
"retentionPeriod" : {
"description": "Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`.",
"$ref": "../../type/basic.json#/definitions/duration"
},
"extension": {
"description": "Entity extension data with custom attributes added to the entity.",
"$ref": "../../type/basic.json#/definitions/entityExtension"
}
},
"required": ["name", "service"],

View File

@ -2,6 +2,7 @@
"$id": "https://open-metadata.org/schema/entity/data/databaseSchema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Database Schema",
"$comment": "@om-entity-type",
"description": "This schema defines the Database Schema entity. A `Database Schema` is collection of tables, views, stored procedures, and other database objects.",
"type": "object",
"javaType": "org.openmetadata.schema.entity.data.DatabaseSchema",
@ -93,6 +94,14 @@
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
},
"retentionPeriod" : {
"description": "Retention period of the data in the database schema. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`. When not set, the retention period is inherited from the parent database, if it exists.",
"$ref": "../../type/basic.json#/definitions/duration"
},
"extension": {
"description": "Entity extension data with custom attributes added to the entity.",
"$ref": "../../type/basic.json#/definitions/entityExtension"
}
},
"required": ["name", "database", "service"],

View File

@ -982,6 +982,10 @@
"type": "boolean",
"default": false
},
"retentionPeriod" : {
"description": "Retention period of the data in the table. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`. When not set, the retention period is inherited from the parent database schema, if it exists.",
"$ref": "../../type/basic.json#/definitions/duration"
},
"extension": {
"description": "Entity extension data with custom attributes added to the entity.",
"$ref": "../../type/basic.json#/definitions/entityExtension"