From 0ba22c1d2f64f5c0b5e08337e1b161728261aa97 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 16 May 2022 15:26:20 -0700 Subject: [PATCH] Fixes #4977 Add APIs for adding custom fields to an existing entity (#4978) --- .../v002__create_db_connection_info.sql | 14 +- .../v002__create_db_connection_info.sql | 14 +- catalog-rest-service/pom.xml | 3 + .../CatalogGenericExceptionMapper.java | 1 - .../catalog/jdbi3/CollectionDAO.java | 84 ++++++----- .../catalog/jdbi3/EntityRepository.java | 22 ++- .../catalog/jdbi3/FeedRepository.java | 46 +++---- .../catalog/jdbi3/RoleRepository.java | 16 --- .../catalog/jdbi3/TypeRepository.java | 130 +++++++++++++++--- .../catalog/resources/types/TypeResource.java | 71 +++++++--- .../openmetadata/catalog/util/EntityUtil.java | 5 + .../openmetadata/catalog/util/JsonUtils.java | 76 +++++++++- .../resources/json/schema/api/createType.json | 9 +- .../json/schema/entity/data/table.json | 1 + .../resources/json/schema/entity/type.json | 81 +++++++++-- .../resources/json/schema/type/basic.json | 22 +-- .../catalog/CatalogApplicationTest.java | 1 + .../catalog/resources/EntityResourceTest.java | 8 +- .../resources/metadata/TypeResourceTest.java | 74 +++++++++- 19 files changed, 501 insertions(+), 177 deletions(-) diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v002__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v002__create_db_connection_info.sql index 3f12882de91..9302363e32d 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v002__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v002__create_db_connection_info.sql @@ -1,18 +1,12 @@ --- --- Table to be used for generic entities that are limited in number. Examples of generic entities are --- Attribute entity, domain entities etc. --- --- This reduces need for defining a table per entity. --- -CREATE TABLE IF NOT EXISTS generic_entity ( +CREATE TABLE IF NOT EXISTS type_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, - -- Fully qualified name formed by entityType + "." + entityName - fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, + category VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.category') NOT NULL, json JSON NOT NULL, updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, PRIMARY KEY (id), - UNIQUE (fullyQualifiedName) + UNIQUE (name) ); ALTER TABLE webhook_entity diff --git a/bootstrap/sql/org.postgresql.Driver/v002__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v002__create_db_connection_info.sql index d17120719b6..98c97f6cd08 100644 --- a/bootstrap/sql/org.postgresql.Driver/v002__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v002__create_db_connection_info.sql @@ -1,18 +1,12 @@ --- --- Table to be used for generic entities that are limited in number. Examples of generic entities are --- Attribute entity, domain entities etc. --- --- This reduces need for defining a table per entity. --- -CREATE TABLE IF NOT EXISTS generic_entity ( +CREATE TABLE IF NOT EXISTS type_entity ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, - -- Fully qualified name formed by entityType + "." + entityName - fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, + category VARCHAR(256) GENERATED ALWAYS AS (json ->> 'category') STORED NOT NULL, json JSONB NOT NULL, updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, PRIMARY KEY (id), - UNIQUE (fullyQualifiedName) + UNIQUE (name) ); ALTER TABLE webhook_entity diff --git a/catalog-rest-service/pom.xml b/catalog-rest-service/pom.xml index db8b9e07649..d884616571b 100644 --- a/catalog-rest-service/pom.xml +++ b/catalog-rest-service/pom.xml @@ -486,6 +486,9 @@ org.apache.maven.plugins maven-surefire-plugin ${maven-surefire.version} + + false + default-test diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogGenericExceptionMapper.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogGenericExceptionMapper.java index f6c7c4fed4f..9652dfe6c9b 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogGenericExceptionMapper.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogGenericExceptionMapper.java @@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory; public class CatalogGenericExceptionMapper implements ExceptionMapper { @Override public Response toResponse(Throwable ex) { - ex.printStackTrace(); LOG.debug(ex.getMessage()); if (ex instanceof ProcessingException || ex instanceof IllegalArgumentException) { final Response response = BadRequestException.of().getResponse(); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java index d9c733bae4d..6a7ec13fe90 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java @@ -176,7 +176,7 @@ public interface CollectionDAO { WebhookDAO webhookDAO(); @CreateSqlObject - GenericEntityDAO genericEntityDAO(); + TypeEntityDAO typeEntityDAO(); interface DashboardDAO extends EntityDAO { @Override @@ -339,16 +339,23 @@ public interface CollectionDAO { interface EntityRelationshipDAO { default int insert(UUID fromId, UUID toId, String fromEntity, String toEntity, int relation) { - return insert(fromId.toString(), toId.toString(), fromEntity, toEntity, relation); + return insert(fromId, toId, fromEntity, toEntity, relation, null); + } + + default int insert(UUID fromId, UUID toId, String fromEntity, String toEntity, int relation, String json) { + return insert(fromId.toString(), toId.toString(), fromEntity, toEntity, relation, json); } @ConnectionAwareSqlUpdate( value = - "INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation)", + "INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) " + + "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation)", connectionType = MYSQL) @ConnectionAwareSqlUpdate( value = - "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation) ON CONFLICT (fromId, toId, relation) DO NOTHING", + "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) " + + "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation) " + + "ON CONFLICT (fromId, toId, relation) DO NOTHING", connectionType = POSTGRES) int insert( @Bind("fromId") String fromId, @@ -357,6 +364,25 @@ public interface CollectionDAO { @Bind("toEntity") String toEntity, @Bind("relation") int relation); + @ConnectionAwareSqlUpdate( + value = + "INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation, json) " + + "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation, :json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation, json) VALUES " + + "(:fromId, :toId, :fromEntity, :toEntity, :relation, (:json :: jsonb)) " + + "ON CONFLICT (fromId, toId, relation) DO NOTHING", + connectionType = POSTGRES) + int insert( + @Bind("fromId") String fromId, + @Bind("toId") String toId, + @Bind("fromEntity") String fromEntity, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation, + @Bind("json") String json); + // // Find to operations // @@ -368,17 +394,6 @@ public interface CollectionDAO { List findTo( @Bind("fromId") String fromId, @Bind("fromEntity") String fromEntity, @Bind("relation") int relation); - @SqlQuery( - "SELECT toId, toEntity FROM entity_relationship " - + "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity " - + "ORDER BY toId") - @RegisterRowMapper(ToEntityReferenceMapper.class) - List findToReference( - @Bind("fromId") String fromId, - @Bind("fromEntity") String fromEntity, - @Bind("relation") int relation, - @Bind("toEntity") String toEntity); - @SqlQuery( "SELECT toId FROM entity_relationship " + "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity " @@ -421,17 +436,6 @@ public interface CollectionDAO { List findFrom( @Bind("toId") String toId, @Bind("toEntity") String toEntity, @Bind("relation") int relation); - @SqlQuery( - "SELECT fromId, fromEntity FROM entity_relationship " - + "WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation AND fromEntity = :fromEntity " - + "ORDER BY fromId") - @RegisterRowMapper(FromEntityReferenceMapper.class) - List findFromEntity( - @Bind("toId") String toId, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation, - @Bind("fromEntity") String fromEntity); - // // Delete Operations // @@ -709,18 +713,22 @@ public interface CollectionDAO { interface FieldRelationshipDAO { @ConnectionAwareSqlUpdate( value = - "INSERT IGNORE INTO field_relationship(fromFQN, toFQN, fromType, toType, relation) VALUES (:fromFQN, :toFQN, :fromType, :toType, :relation)", + "INSERT IGNORE INTO field_relationship(fromFQN, toFQN, fromType, toType, relation, json) " + + "VALUES (:fromFQN, :toFQN, :fromType, :toType, :relation, :json)", connectionType = MYSQL) @ConnectionAwareSqlUpdate( value = - "INSERT INTO field_relationship(fromFQN, toFQN, fromType, toType, relation) VALUES (:fromFQN, :toFQN, :fromType, :toType, :relation) ON CONFLICT (fromFQN, toFQN, relation) DO NOTHING", + "INSERT INTO field_relationship(fromFQN, toFQN, fromType, toType, relation, json) " + + "VALUES (:fromFQN, :toFQN, :fromType, :toType, :relation, (:json :: jsonb)) " + + "ON CONFLICT (fromFQN, toFQN, relation) DO NOTHING", connectionType = POSTGRES) - void insert( + int insert( @Bind("fromFQN") String fromFQN, @Bind("toFQN") String toFQN, @Bind("fromType") String fromType, @Bind("toType") String toType, - @Bind("relation") int relation); + @Bind("relation") int relation, + @Bind("json") String json); @ConnectionAwareSqlUpdate( value = @@ -794,6 +802,16 @@ public interface CollectionDAO { @SqlUpdate("DELETE from field_relationship ") void deleteAllByPrefixInternal(@Define("cond") String cond); + @SqlUpdate( + "DELETE from field_relationship WHERE fromFQN = :fromFQN AND toFQN = :toFQN AND fromType = :fromType " + + "AND toType = :toType AND relation = :relation") + void delete( + @Bind("fromFQN") String fromFQN, + @Bind("toFQN") String toFQN, + @Bind("fromType") String fromType, + @Bind("toType") String toType, + @Bind("relation") int relation); + class ToFieldMapper implements RowMapper> { @Override public List map(ResultSet rs, StatementContext ctx) throws SQLException { @@ -1560,10 +1578,10 @@ public interface CollectionDAO { List listWithoutEntityFilter(@Bind("eventType") String eventType, @Bind("timestamp") long timestamp); } - interface GenericEntityDAO extends EntityDAO { + interface TypeEntityDAO extends EntityDAO { @Override default String getTableName() { - return "generic_entity"; + return "type_entity"; } @Override @@ -1573,7 +1591,7 @@ public interface CollectionDAO { @Override default String getNameColumn() { - return "fullyQualifiedName"; + return "name"; } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java index 5bde058dd39..c3bcf615c6f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java @@ -115,7 +115,7 @@ import org.openmetadata.catalog.util.ResultList; public abstract class EntityRepository { private final String collectionPath; private final Class entityClass; - private final String entityType; + protected final String entityType; public final EntityDAO dao; protected final CollectionDAO daoCollection; protected final List allowedFields; @@ -129,7 +129,7 @@ public abstract class EntityRepository { private final Fields patchFields; /** Fields that can be updated during PUT operation */ - private final Fields putFields; + protected final Fields putFields; EntityRepository( String collectionPath, @@ -737,8 +737,24 @@ public abstract class EntityRepository { return addRelationship(fromId, toId, fromEntity, toEntity, relationship, false); } + public int addRelationship( + UUID fromId, UUID toId, String fromEntity, String toEntity, Relationship relationship, String json) { + return addRelationship(fromId, toId, fromEntity, toEntity, relationship, json, false); + } + public int addRelationship( UUID fromId, UUID toId, String fromEntity, String toEntity, Relationship relationship, boolean bidirectional) { + return addRelationship(fromId, toId, fromEntity, toEntity, relationship, null, bidirectional); + } + + public int addRelationship( + UUID fromId, + UUID toId, + String fromEntity, + String toEntity, + Relationship relationship, + String json, + boolean bidirectional) { UUID from = fromId; UUID to = toId; if (bidirectional && fromId.compareTo(toId) > 0) { @@ -747,7 +763,7 @@ public abstract class EntityRepository { from = toId; to = fromId; } - return daoCollection.relationshipDAO().insert(from, to, fromEntity, toEntity, relationship.ordinal()); + return daoCollection.relationshipDAO().insert(fromId, toId, fromEntity, toEntity, relationship.ordinal(), json); } public List findBoth(UUID entity1, String entityType1, Relationship relationship, String entity2) { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/FeedRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/FeedRepository.java index 9e2d59f9a64..d1faef5a269 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/FeedRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/FeedRepository.java @@ -13,6 +13,11 @@ package org.openmetadata.catalog.jdbi3; +import static org.openmetadata.catalog.type.Relationship.ADDRESSED_TO; +import static org.openmetadata.catalog.type.Relationship.CREATED; +import static org.openmetadata.catalog.type.Relationship.IS_ABOUT; +import static org.openmetadata.catalog.type.Relationship.REPLIED_TO; + import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; import java.util.ArrayList; @@ -83,13 +88,7 @@ public class FeedRepository { dao.feedDAO().insert(JsonUtils.pojoToJson(thread)); // Add relationship User -- created --> Thread relationship - dao.relationshipDAO() - .insert( - createdByUser.getId().toString(), - thread.getId().toString(), - Entity.USER, - Entity.THREAD, - Relationship.CREATED.ordinal()); + dao.relationshipDAO().insert(createdByUser.getId(), thread.getId(), Entity.USER, Entity.THREAD, CREATED.ordinal()); // Add field relationship data asset Thread -- isAbout ---> entity/entityField // relationship @@ -99,17 +98,13 @@ public class FeedRepository { about.getFullyQualifiedFieldValue(), // to FQN Entity.THREAD, // From type about.getFullyQualifiedFieldType(), // to Type - Relationship.IS_ABOUT.ordinal()); + IS_ABOUT.ordinal(), + null); // Add the owner also as addressedTo as the entity he owns when addressed, the owner is actually being addressed if (entityOwner != null) { dao.relationshipDAO() - .insert( - thread.getId().toString(), - entityOwner.getId().toString(), - Entity.THREAD, - entityOwner.getType(), - Relationship.ADDRESSED_TO.ordinal()); + .insert(thread.getId(), entityOwner.getId(), Entity.THREAD, entityOwner.getType(), ADDRESSED_TO.ordinal()); } // Add mentions to field relationship table @@ -151,7 +146,8 @@ public class FeedRepository { thread.getId().toString(), mention.getFullyQualifiedFieldType(), Entity.THREAD, - Relationship.MENTIONED_IN.ordinal())); + Relationship.MENTIONED_IN.ordinal(), + null)); } @Transaction @@ -178,13 +174,7 @@ public class FeedRepository { } } if (!relationAlreadyExists) { - dao.relationshipDAO() - .insert( - fromUser.getId().toString(), - thread.getId().toString(), - Entity.USER, - Entity.THREAD, - Relationship.REPLIED_TO.ordinal()); + dao.relationshipDAO().insert(fromUser.getId(), thread.getId(), Entity.USER, Entity.THREAD, REPLIED_TO.ordinal()); } // Add mentions into field relationship table @@ -233,7 +223,7 @@ public class FeedRepository { result = dao.feedDAO() .listCountByEntityLink( - StringUtils.EMPTY, Entity.THREAD, StringUtils.EMPTY, Relationship.IS_ABOUT.ordinal(), isResolved); + StringUtils.EMPTY, Entity.THREAD, StringUtils.EMPTY, IS_ABOUT.ordinal(), isResolved); } else { EntityLink entityLink = EntityLink.parse(link); EntityReference reference = EntityUtil.validateEntityLink(entityLink); @@ -253,7 +243,7 @@ public class FeedRepository { entityLink.getFullyQualifiedFieldValue(), Entity.THREAD, entityLink.getFullyQualifiedFieldType(), - Relationship.IS_ABOUT.ordinal(), + IS_ABOUT.ordinal(), isResolved); } } @@ -333,7 +323,7 @@ public class FeedRepository { limit + 1, time, isResolved, - Relationship.IS_ABOUT.ordinal()); + IS_ABOUT.ordinal()); } else { jsons = dao.feedDAO() @@ -343,7 +333,7 @@ public class FeedRepository { limit + 1, time, isResolved, - Relationship.IS_ABOUT.ordinal()); + IS_ABOUT.ordinal()); } threads = JsonUtils.readObjects(jsons, Thread.class); total = @@ -352,7 +342,7 @@ public class FeedRepository { entityLink.getFullyQualifiedFieldValue(), entityLink.getFullyQualifiedFieldType(), isResolved, - Relationship.IS_ABOUT.ordinal()); + IS_ABOUT.ordinal()); } } else { FilteredThreads filteredThreads; @@ -474,7 +464,7 @@ public class FeedRepository { String userId, int limit, long time, boolean isResolved, PaginationType paginationType) throws IOException { List teams = EntityUtil.populateEntityReferences( - dao.relationshipDAO().findFromEntity(userId, Entity.USER, Relationship.HAS.ordinal(), Entity.TEAM)); + dao.relationshipDAO().findFrom(userId, Entity.USER, Relationship.HAS.ordinal(), Entity.TEAM), Entity.TEAM); List teamNames = teams.stream().map(EntityReference::getName).collect(Collectors.toList()); if (teamNames.isEmpty()) { teamNames = List.of(StringUtils.EMPTY); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/RoleRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/RoleRepository.java index aac2292b76f..7e695d3d021 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/RoleRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/RoleRepository.java @@ -25,9 +25,6 @@ import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.entity.teams.Role; -import org.openmetadata.catalog.entity.teams.User; -import org.openmetadata.catalog.exception.CatalogExceptionMessage; -import org.openmetadata.catalog.exception.EntityNotFoundException; import org.openmetadata.catalog.resources.teams.RoleResource; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.Relationship; @@ -178,18 +175,5 @@ public class RoleRepository extends EntityRepository { .relationshipDAO() .deleteTo(role.getId().toString(), Entity.ROLE, Relationship.HAS.ordinal(), Entity.USER); } - - private List getAllUsers() { - EntityRepository userRepository = Entity.getEntityRepository(Entity.USER); - try { - // Assumptions: - // - we will not have more than Integer.MAX_VALUE users in the system. - // - we do not need to update deleted user's roles. - ListFilter filter = new ListFilter(); - return userRepository.listAfter(null, Fields.EMPTY_FIELDS, filter, Integer.MAX_VALUE - 1, null).getData(); - } catch (IOException e) { - throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entitiesNotFound(Entity.USER)); - } - } } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TypeRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TypeRepository.java index 7d3dbea30bb..48e3ce22f09 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TypeRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TypeRepository.java @@ -16,34 +16,43 @@ package org.openmetadata.catalog.jdbi3; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; + +import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import javax.ws.rs.core.UriInfo; +import lombok.extern.slf4j.Slf4j; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.entity.Type; +import org.openmetadata.catalog.entity.type.Category; +import org.openmetadata.catalog.entity.type.CustomField; import org.openmetadata.catalog.resources.types.TypeResource; +import org.openmetadata.catalog.type.Include; +import org.openmetadata.catalog.type.Relationship; +import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.FullyQualifiedName; +import org.openmetadata.catalog.util.JsonUtils; +import org.openmetadata.catalog.util.RestUtil.PutResponse; +@Slf4j public class TypeRepository extends EntityRepository { - // TODO fix this - private static final String UPDATE_FIELDS = ""; - private static final String PATCH_FIELDS = ""; + private static final String UPDATE_FIELDS = "customFields"; + private static final String PATCH_FIELDS = "customFields"; public TypeRepository(CollectionDAO dao) { - super( - TypeResource.COLLECTION_PATH, - Entity.TYPE, - Type.class, - dao.genericEntityDAO(), - dao, - PATCH_FIELDS, - UPDATE_FIELDS); + super(TypeResource.COLLECTION_PATH, Entity.TYPE, Type.class, dao.typeEntityDAO(), dao, PATCH_FIELDS, UPDATE_FIELDS); allowEdits = true; } @Override - public Type setFields(Type attribute, Fields fields) throws IOException { - return attribute; + public Type setFields(Type type, Fields fields) throws IOException { + type.withCustomFields(fields.contains("customFields") ? getCustomFields(type) : null); + return type; } @Override @@ -54,9 +63,10 @@ public class TypeRepository extends EntityRepository { @Override public void storeEntity(Type type, boolean update) throws IOException { URI href = type.getHref(); - type.withHref(null); + List customFields = type.getCustomFields(); + type.withHref(null).withCustomFields(null); store(type.getId(), type, update); - type.withHref(href); + type.withHref(href).withCustomFields(customFields); } @Override @@ -65,19 +75,95 @@ public class TypeRepository extends EntityRepository { } @Override - public void setFullyQualifiedName(Type entity) { - entity.setFullyQualifiedName(FullyQualifiedName.build(Entity.TYPE, entity.getNameSpace(), entity.getName())); + public EntityUpdater getUpdater(Type original, Type updated, Operation operation) { + return new TypeUpdater(original, updated, operation); } - @Override - public EntityUpdater getUpdater(Type original, Type updated, Operation operation) { - return new AttributeUpdater(original, updated, operation); + public PutResponse addCustomField(UriInfo uriInfo, String updatedBy, String id, CustomField field) + throws IOException { + Type type = dao.findEntityById(UUID.fromString(id), Include.NON_DELETED); + if (type.getCategory().equals(Category.Field)) { + throw new IllegalArgumentException("Field types can't be extended"); + } + setFields(type, putFields); + + dao.findEntityById(field.getFieldType().getId()); // Validate customField type exists + type.getCustomFields().add(field); + type.setUpdatedBy(updatedBy); + type.setUpdatedAt(System.currentTimeMillis()); + return createOrUpdate(uriInfo, type); + } + + private String getCustomFieldFQNPrefix(Type type) { + return FullyQualifiedName.build(type.getName(), "customFields"); + } + + private String getCustomFieldFQN(Type type, String fieldName) { + return FullyQualifiedName.build(type.getName(), "customFields", fieldName); + } + + private List getCustomFields(Type type) throws IOException { + List customFields = new ArrayList<>(); + List> results = + daoCollection + .fieldRelationshipDAO() + .listToByPrefix(getCustomFieldFQNPrefix(type), Entity.TYPE, Entity.TYPE, Relationship.HAS.ordinal()); + for (List result : results) { + CustomField field = JsonUtils.readValue(result.get(2), CustomField.class); + field.setFieldType(dao.findEntityReferenceByName(result.get(1))); + customFields.add(field); + } + customFields.sort(EntityUtil.compareCustomField); + return customFields; } /** Handles entity updated from PUT and POST operation. */ - public class AttributeUpdater extends EntityUpdater { - public AttributeUpdater(Type original, Type updated, Operation operation) { + public class TypeUpdater extends EntityUpdater { + public TypeUpdater(Type original, Type updated, Operation operation) { super(original, updated, operation); } + + @Override + public void entitySpecificUpdate() throws IOException { + updateCustomFields(); + } + + private void updateCustomFields() throws JsonProcessingException { + List updatedFields = listOrEmpty(updated.getCustomFields()); + List origFields = listOrEmpty(original.getCustomFields()); + List added = new ArrayList<>(); + List deleted = new ArrayList<>(); + recordListChange("charts", origFields, updatedFields, added, deleted, EntityUtil.customFieldMatch); + for (CustomField field : added) { + String customFieldFQN = getCustomFieldFQN(updated, field.getName()); + String customFieldJson = JsonUtils.pojoToJson(field); + LOG.info( + "Adding customField {} with type {} to the entity {}", + field.getName(), + field.getFieldType().getName(), + updated.getName()); + daoCollection + .fieldRelationshipDAO() + .insert( + customFieldFQN, + field.getFieldType().getName(), + Entity.TYPE, + Entity.TYPE, + Relationship.HAS.ordinal(), + customFieldJson); + } + for (CustomField field : deleted) { + String customFieldFQN = getCustomFieldFQN(updated, field.getName()); + LOG.info( + "Deleting customField {} with type {} from the entity {}", + field.getName(), + field.getFieldType().getName(), + updated.getName()); + daoCollection + .fieldRelationshipDAO() + .delete( + customFieldFQN, field.getFieldType().getName(), Entity.TYPE, Entity.TYPE, Relationship.HAS.ordinal()); + } + } } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/types/TypeResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/types/TypeResource.java index 884163d7a71..7727d7aad55 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/types/TypeResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/types/TypeResource.java @@ -16,6 +16,7 @@ package org.openmetadata.catalog.resources.types; import static org.openmetadata.catalog.security.SecurityUtil.ADMIN; import static org.openmetadata.catalog.security.SecurityUtil.BOT; import static org.openmetadata.catalog.security.SecurityUtil.OWNER; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import com.google.inject.Inject; import io.swagger.annotations.Api; @@ -52,18 +53,22 @@ import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; import org.openmetadata.catalog.CatalogApplicationConfig; +import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.api.CreateType; import org.openmetadata.catalog.entity.Type; +import org.openmetadata.catalog.entity.type.Category; +import org.openmetadata.catalog.entity.type.CustomField; import org.openmetadata.catalog.jdbi3.CollectionDAO; import org.openmetadata.catalog.jdbi3.ListFilter; import org.openmetadata.catalog.jdbi3.TypeRepository; import org.openmetadata.catalog.resources.Collection; import org.openmetadata.catalog.resources.EntityResource; import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.security.SecurityUtil; import org.openmetadata.catalog.type.EntityHistory; import org.openmetadata.catalog.type.Include; -import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.JsonUtils; +import org.openmetadata.catalog.util.RestUtil.PutResponse; import org.openmetadata.catalog.util.ResultList; @Path("/v1/metadata/types") @@ -77,7 +82,8 @@ public class TypeResource extends EntityResource { @Override public Type addHref(UriInfo uriInfo, Type type) { - return type; // Nothing to do + listOrEmpty(type.getCustomFields()).forEach(field -> Entity.withHref(uriInfo, field.getFieldType())); + return type; } @Inject @@ -88,26 +94,18 @@ public class TypeResource extends EntityResource { @SuppressWarnings("unused") // Method used for reflection public void initialize(CatalogApplicationConfig config) throws IOException { // Find tag definitions and load tag categories from the json file, if necessary - List jsonSchemas = EntityUtil.getJsonDataResources(".*json/schema/type/.*\\.json$"); long now = System.currentTimeMillis(); - for (String jsonSchema : jsonSchemas) { - try { - List types = JsonUtils.getTypes(jsonSchema); - types.forEach( - type -> { - type.withId(UUID.randomUUID()).withUpdatedBy("admin").withUpdatedAt(now); - LOG.info("Loading from {} type {} with schema {}", jsonSchema, type.getName(), type.getSchema()); - try { - this.dao.createOrUpdate(null, type); - } catch (IOException e) { - LOG.error("Error loading type {} from {}", type.getName(), jsonSchema, e); - e.printStackTrace(); - } - }); - } catch (Exception e) { - LOG.warn("Failed to initialize the types from jsonSchema file {}", jsonSchema, e); - } - } + List types = JsonUtils.getTypes(); + types.forEach( + type -> { + type.withId(UUID.randomUUID()).withUpdatedBy("admin").withUpdatedAt(now); + LOG.info("Loading type {} with schema {}", type.getName(), type.getSchema()); + try { + this.dao.createOrUpdate(null, type); + } catch (IOException e) { + LOG.error("Error loading type {}", type.getName(), e); + } + }); } public static class TypeList extends ResultList { @@ -121,7 +119,7 @@ public class TypeResource extends EntityResource { } } - public static final String FIELDS = ""; + public static final String FIELDS = "customFields"; @GET @Valid @@ -141,6 +139,11 @@ public class TypeResource extends EntityResource { public ResultList list( @Context UriInfo uriInfo, @Context SecurityContext securityContext, + @Parameter( + description = "Filter types by metadata type category.", + schema = @Schema(type = "string", example = "Field, Entity")) + @QueryParam("category") + Category category, @Parameter(description = "Limit the number types returned. (1 to 1000000, " + "default = 10)") @DefaultValue("10") @Min(0) @@ -349,11 +352,35 @@ public class TypeResource extends EntityResource { return delete(uriInfo, securityContext, id, false, true, ADMIN | BOT); } + @PUT + @Path("/{id}") + @Operation( + summary = "Add a field to an entity", + tags = "metadata", + description = "Add a field to an entity type. Fields can only be added to entity type and not field type.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "type for instance {id} is not found") + }) + public Response addField( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Type Id", schema = @Schema(type = "string")) @PathParam("id") String id, + CustomField field) + throws IOException { + SecurityUtil.authorizeAdmin(authorizer, securityContext, ADMIN | BOT); + PutResponse response = dao.addCustomField(uriInfo, securityContext.getUserPrincipal().getName(), id, field); + addHref(uriInfo, response.getEntity()); + return response.toResponse(); + } + private Type getType(SecurityContext securityContext, CreateType create) { return new Type() .withId(UUID.randomUUID()) .withName(create.getName()) + .withFullyQualifiedName(create.getName()) .withDisplayName(create.getDisplayName()) + .withCategory(create.getCategory()) .withSchema(create.getSchema()) .withDescription(create.getDescription()) .withUpdatedBy(securityContext.getUserPrincipal().getName()) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java index d3b2560b737..145c77b8284 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java @@ -40,6 +40,7 @@ import org.openmetadata.catalog.EntityInterface; import org.openmetadata.catalog.api.data.TermReference; import org.openmetadata.catalog.entity.data.GlossaryTerm; import org.openmetadata.catalog.entity.data.Table; +import org.openmetadata.catalog.entity.type.CustomField; import org.openmetadata.catalog.exception.CatalogExceptionMessage; import org.openmetadata.catalog.exception.EntityNotFoundException; import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityVersionPair; @@ -80,6 +81,7 @@ public final class EntityUtil { Comparator.comparing(TableConstraint::getConstraintType); public static final Comparator compareChangeEvent = Comparator.comparing(ChangeEvent::getTimestamp); public static final Comparator compareGlossaryTerm = Comparator.comparing(GlossaryTerm::getName); + public static final Comparator compareCustomField = Comparator.comparing(CustomField::getName); // // Matchers used for matching two items in a list @@ -130,6 +132,9 @@ public final class EntityUtil { public static final BiPredicate termReferenceMatch = (ref1, ref2) -> ref1.getName().equals(ref2.getName()) && ref1.getEndpoint().equals(ref2.getEndpoint()); + public static final BiPredicate customFieldMatch = + (ref1, ref2) -> ref1.getName().equals(ref2.getName()); + private EntityUtil() {} /** Validate Ingestion Schedule */ diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonUtils.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonUtils.java index 6bf9484e26a..0952c5a29d1 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonUtils.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonUtils.java @@ -45,10 +45,14 @@ import javax.json.JsonReader; import javax.json.JsonStructure; import javax.json.JsonValue; import javax.ws.rs.core.MediaType; +import lombok.extern.slf4j.Slf4j; import org.openmetadata.catalog.entity.Type; +import org.openmetadata.catalog.entity.type.Category; +@Slf4j public final class JsonUtils { - public static final String TYPE_ANNOTATION = "@om-type"; + public static final String FIELD_TYPE_ANNOTATION = "@om-field-type"; + public static final String ENTITY_TYPE_ANNOTATION = "@om-entity-type"; public static final MediaType DEFAULT_MEDIA_TYPE = MediaType.APPLICATION_JSON_TYPE; private static final ObjectMapper OBJECT_MAPPER; private static final JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(VersionFlag.V7); @@ -217,11 +221,39 @@ public final class JsonUtils { return comment != null && comment.contains(annotation); } + /** Get all the fields types and entity types from OpenMetadata JSON schema definition files. */ + public static List getTypes() throws IOException { + // Get Field Types + List types = new ArrayList<>(); + List jsonSchemas = EntityUtil.getJsonDataResources(".*json/schema/type/.*\\.json$"); + for (String jsonSchema : jsonSchemas) { + try { + types.addAll(JsonUtils.getFieldTypes(jsonSchema)); + } catch (Exception e) { + LOG.warn("Failed to initialize the types from jsonSchema file {}", jsonSchema, e); + } + } + + // Get Entity Types + jsonSchemas = EntityUtil.getJsonDataResources(".*json/schema/entity/.*\\.json$"); + for (String jsonSchema : jsonSchemas) { + try { + Type entityType = JsonUtils.getEntityType(jsonSchema); + if (entityType != null) { + types.add(entityType); + } + } catch (Exception e) { + LOG.warn("Failed to initialize the types from jsonSchema file {}", jsonSchema, e); + } + } + return types; + } + /** - * Get all the types from the `definitions` section of a JSON schema file that are annotated with "$comment" field set - * to "@om-type". + * Get all the fields types from the `definitions` section of a JSON schema file that are annotated with "$comment" + * field set to "@om-field-type". */ - public static List getTypes(String jsonSchemaFile) throws IOException { + public static List getFieldTypes(String jsonSchemaFile) throws IOException { JsonNode node = OBJECT_MAPPER.readTree( Objects.requireNonNull(JsonUtils.class.getClassLoader().getResourceAsStream(jsonSchemaFile))); @@ -236,12 +268,15 @@ public final class JsonUtils { Iterator> definitions = node.get("definitions").fields(); while (definitions != null && definitions.hasNext()) { Entry entry = definitions.next(); + String typeName = entry.getKey(); JsonNode value = entry.getValue(); - if (JsonUtils.hasAnnotation(value, JsonUtils.TYPE_ANNOTATION)) { + if (JsonUtils.hasAnnotation(value, JsonUtils.FIELD_TYPE_ANNOTATION)) { String description = String.valueOf(value.get("description")); Type type = new Type() - .withName(entry.getKey()) + .withName(typeName) + .withCategory(Category.Field) + .withFullyQualifiedName(typeName) .withNameSpace(jsonNamespace) .withDescription(description) .withDisplayName(entry.getKey()) @@ -251,4 +286,33 @@ public final class JsonUtils { } return types; } + + /** + * Get all the fields types from the `definitions` section of a JSON schema file that are annotated with "$comment" + * field set to "@om-entity-type". + */ + public static Type getEntityType(String jsonSchemaFile) throws IOException { + JsonNode node = + OBJECT_MAPPER.readTree( + Objects.requireNonNull(JsonUtils.class.getClassLoader().getResourceAsStream(jsonSchemaFile))); + if (!JsonUtils.hasAnnotation(node, JsonUtils.ENTITY_TYPE_ANNOTATION)) { + return null; + } + + String fileName = Paths.get(jsonSchemaFile).getFileName().toString(); + String entityName = fileName.replace(" ", "").replace(".json", ""); + + String namespaceFile = Paths.get(jsonSchemaFile).getParent().getFileName().toString(); + String namespace = namespaceFile.replace(" ", "").replace(".json", ""); + + String description = String.valueOf(node.get("description")); + return new Type() + .withName(entityName) + .withCategory(Category.Entity) + .withFullyQualifiedName(entityName) + .withNameSpace(namespace) + .withDescription(description) + .withDisplayName(entityName) + .withSchema(node.toPrettyString()); + } } diff --git a/catalog-rest-service/src/main/resources/json/schema/api/createType.json b/catalog-rest-service/src/main/resources/json/schema/api/createType.json index b31c1fcf63c..4e1d41ba7fd 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/createType.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/createType.json @@ -1,5 +1,5 @@ { - "$id": "https://open-metadata.org/schema/api/data/createAttribute.json", + "$id": "https://open-metadata.org/schema/api/data/createType.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "createType", "description": "Create a Type to be used for extending entities.", @@ -7,7 +7,7 @@ "properties": { "name": { "description": "Unique name that identifies a Type.", - "$ref": "../type/basic.json#/definitions/entityName" + "$ref": "../entity/type.json#/definitions/typeName" }, "nameSpace": { "description": "Namespace or group to which this type belongs to.", @@ -20,7 +20,10 @@ }, "description": { "description": "Optional description of the type.", - "type": "string" + "$ref" : "../type/basic.json#/definitions/markdown" + }, + "category" : { + "$ref" : "../entity/type.json#/definitions/category" }, "schema": { "description": "JSON schema encoded as string. This will be used to validate the type values.", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json index c7c677d3e10..65e68fefffb 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json @@ -2,6 +2,7 @@ "$id": "https://open-metadata.org/schema/entity/data/table.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "Table", + "$comment" : "@om-entity-type", "description": "This schema defines the Table entity. A Table organizes data in rows and columns and is defined by a Schema. OpenMetadata does not have a separate abstraction for Schema. Both Table and Schema are captured in this entity.", "type": "object", "javaType": "org.openmetadata.catalog.entity.data.Table", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/type.json b/catalog-rest-service/src/main/resources/json/schema/entity/type.json index 54b72d3e802..695057bc868 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/type.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/type.json @@ -2,10 +2,63 @@ "$id": "https://open-metadata.org/schema/entity/type.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "Type", - "description": "This schema defines a type entity used for extending an entity with custom attributes.", + "description": "This schema defines a type as an entity. Types includes field types and entity types. Custom types can also be defined by the users to extend the metadata system.", "type": "object", "javaType": "org.openmetadata.catalog.entity.Type", "javaInterfaces": ["org.openmetadata.catalog.EntityInterface"], + "definitions": { + "category" : { + "description" : "Metadata category to which a type belongs to.", + "type": "string", + "javaType": "org.openmetadata.catalog.entity.type.Category", + "enum": [ + "field", + "entity" + ], + "javaEnums": [ + { + "name": "Field" + }, + { + "name": "Entity" + } + ] + }, + "fieldName": { + "description": "Name of the entity field. Note a field name must be unique for an entity. Field name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.", + "type" : "string", + "pattern": "^[a-z][a-zA-Z0-9]+$" + }, + "typeName": { + "description": "Name of the field or entity types. Note a field name must be unique for an entity. Field name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.", + "type" : "string", + "pattern": "^[a-z][a-zA-Z0-9]+$" + }, + "customField" : { + "description" : "Type used for adding custom field to an entity to extend it.", + "type" : "object", + "javaType": "org.openmetadata.catalog.entity.type.CustomField", + "properties": { + "name": { + "description": "Name of the entity field. Note a field name must be unique for an entity. Field name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.", + "$ref" : "#/definitions/fieldName" + }, + "description": { + "$ref" : "../type/basic.json#/definitions/markdown" + }, + "fieldType": { + "description": "Reference to a field type. Only field types are allows and entity types are not allowed as custom fields to extend an existing entity", + "$ref": "../type/entityReference.json" + } + }, + "required": [ + "name", + "description", + "fieldType" + ], + "additionalProperties": false + } + }, "properties": { "id": { "description": "Unique identifier of the type instance.", @@ -13,29 +66,39 @@ }, "name": { "description": "Unique name that identifies the type.", - "$ref": "../type/basic.json#/definitions/entityName" + "$ref": "#/definitions/typeName" }, "fullyQualifiedName": { "description": "FullyQualifiedName same as `name`.", "$ref": "../type/basic.json#/definitions/fullyQualifiedEntityName" }, - "nameSpace": { - "description": "Namespace or group to which this type belongs to.", - "type": "string", - "default" : "custom" - }, "displayName": { "description": "Display Name that identifies this type.", "type": "string" }, "description": { "description": "Optional description of entity.", - "type": "string" + "$ref" : "../type/basic.json#/definitions/markdown" + }, + "nameSpace": { + "description": "Namespace or group to which this type belongs to. For example, some of the field types commonly used can come from `basic` namespace. Some of the entities such as `table`, `database`, etc. come from `data` namespace.", + "type": "string", + "default" : "custom" + }, + "category" : { + "$ref" : "#/definitions/category" }, "schema": { "description": "JSON schema encoded as string that defines the type. This will be used to validate the type values.", "$ref": "../type/basic.json#/definitions/jsonSchema" }, + "customFields" : { + "description": "Custom fields added to extend the entity. Only available for entity type", + "type" : "array", + "items" : { + "$ref" : "#/definitions/customField" + } + }, "version": { "description": "Metadata version of the entity.", "$ref": "../type/entityHistory.json#/definitions/entityVersion" @@ -61,7 +124,7 @@ "name", "nameSpace", "description", - "type" + "schema" ], "additionalProperties": false } diff --git a/catalog-rest-service/src/main/resources/json/schema/type/basic.json b/catalog-rest-service/src/main/resources/json/schema/type/basic.json index 1ea3f4f0c56..6011c0ec19c 100644 --- a/catalog-rest-service/src/main/resources/json/schema/type/basic.json +++ b/catalog-rest-service/src/main/resources/json/schema/type/basic.json @@ -5,17 +5,17 @@ "description": "This schema defines basic common types that are used by other schemas.", "definitions": { "integer" : { - "$comment" : "@om-type", + "$comment" : "@om-field-type", "description": "An integer type.", "type" : "integer" }, "number" : { - "$comment" : "@om-type", + "$comment" : "@om-field-type", "description": "A numeric type that includes integer or floating point numbers.", "type" : "integer" }, "string" : { - "$comment" : "@om-type", + "$comment" : "@om-field-type", "description": "A String type.", "type" : "string" }, @@ -25,7 +25,7 @@ "format": "uuid" }, "email": { - "$comment" : "@om-type", + "$comment" : "@om-field-type", "description": "Email address of a user or other entities.", "type": "string", "format": "email", @@ -34,7 +34,7 @@ "maxLength": 127 }, "timestamp": { - "$comment" : "@om-type", + "$comment" : "@om-field-type", "description": "Timestamp in Unix epoch time milliseconds.", "@comment": "Note that during code generation this is converted into long", "type": "integer", @@ -46,7 +46,7 @@ "format": "uri" }, "timeInterval": { - "$comment" : "@om-type", + "$comment" : "@om-field-type", "type": "object", "description": "Time interval in unixTimeMillis.", "javaType": "org.openmetadata.catalog.type.TimeInterval", @@ -63,18 +63,18 @@ "additionalProperties": false }, "duration": { - "$comment" : "@om-type", + "$comment" : "@om-field-type", "description": "Duration in ISO 8601 format in UTC. Example - 'P23DT23H'.", "type": "string" }, "date": { - "$comment" : "@om-type", + "$comment" : "@om-field-type", "description": "Date in ISO 8601 format in UTC. Example - '2018-11-13'.", "type": "string", "format": "date" }, "dateTime": { - "$comment" : "@om-type", + "$comment" : "@om-field-type", "description": "Date and time in ISO 8601 format. Example - '2018-11-13T20:20:39+00:00'.", "type": "string", "format": "date-time" @@ -97,12 +97,12 @@ "maxLength": 256 }, "sqlQuery": { - "$comment" : "@om-type", + "$comment" : "@om-field-type", "description": "SQL query statement. Example - 'select * from orders'.", "type": "string" }, "markdown": { - "$comment" : "@om-type", + "$comment" : "@om-field-type", "description": "Text in Markdown format", "type": "string" }, diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/CatalogApplicationTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/CatalogApplicationTest.java index 74f70754136..6a1c94542de 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/CatalogApplicationTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/CatalogApplicationTest.java @@ -49,6 +49,7 @@ public abstract class CatalogApplicationTest { // The system properties are provided by maven-surefire for testing with mysql and postgres final String jdbcContainerClassName = System.getProperty("jdbcContainerClassName"); final String jdbcContainerImage = System.getProperty("jdbcContainerImage"); + LOG.info("Using test container class {} and image {}", jdbcContainerClassName, jdbcContainerImage); SQL_CONTAINER = (JdbcDatabaseContainer) diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java index 4c79738cfcd..ccfc0318f4d 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java @@ -145,6 +145,7 @@ public abstract class EntityResourceTest extends C protected boolean supportsAuthorizedMetadataOperations = true; protected boolean supportsFieldsQueryParam = true; protected boolean supportsEmptyDescription = true; + protected boolean supportsNameWithDot = true; public static final String DATA_STEWARD_ROLE_NAME = "DataSteward"; public static final String DATA_CONSUMER_ROLE_NAME = "DataConsumer"; @@ -623,14 +624,14 @@ public abstract class EntityResourceTest extends C // Common entity tests for POST operations /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @Test - void post_entityCreateWithInvalidName_400() { + protected void post_entityCreateWithInvalidName_400() { // Create an entity with mandatory name field null final K request = createRequest(null, "description", "displayName", null); assertResponse(() -> createEntity(request, ADMIN_AUTH_HEADERS), BAD_REQUEST, "[name must not be null]"); // Create an entity with mandatory name field empty final K request1 = createRequest("", "description", "displayName", null); - assertResponse(() -> createEntity(request1, ADMIN_AUTH_HEADERS), BAD_REQUEST, ENTITY_NAME_LENGTH_ERROR); + assertResponseContains(() -> createEntity(request1, ADMIN_AUTH_HEADERS), BAD_REQUEST, ENTITY_NAME_LENGTH_ERROR); // Create an entity with mandatory name field too long final K request2 = createRequest(LONG_ENTITY_NAME, "description", "displayName", null); @@ -698,6 +699,9 @@ public abstract class EntityResourceTest extends C @Test void post_entityWithDots_200() throws HttpResponseException { + if (!supportsNameWithDot) { + return; + } // Entity without "." should not have quoted fullyQualifiedName String name = format("%s_foo_bar", entityType); K request = createRequest(name, "", null, null); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/metadata/TypeResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/metadata/TypeResourceTest.java index d71c80dda21..1aa57307eab 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/metadata/TypeResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/metadata/TypeResourceTest.java @@ -14,20 +14,30 @@ package org.openmetadata.catalog.resources.metadata; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS; +import static org.openmetadata.catalog.util.TestUtils.assertResponse; +import static org.openmetadata.catalog.util.TestUtils.assertResponseContains; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import java.io.IOException; import java.net.URISyntaxException; import java.util.Map; +import java.util.UUID; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response.Status; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.TestMethodOrder; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.api.CreateType; import org.openmetadata.catalog.entity.Type; +import org.openmetadata.catalog.entity.type.Category; +import org.openmetadata.catalog.entity.type.CustomField; import org.openmetadata.catalog.resources.EntityResourceTest; import org.openmetadata.catalog.resources.types.TypeResource; import org.openmetadata.catalog.resources.types.TypeResource.TypeList; @@ -43,12 +53,63 @@ public class TypeResourceTest extends EntityResourceTest { super(Entity.TYPE, Type.class, TypeList.class, "metadata/types", TypeResource.FIELDS); supportsEmptyDescription = false; supportsFieldsQueryParam = false; + supportsNameWithDot = false; } @BeforeAll public void setup(TestInfo test) throws IOException, URISyntaxException { super.setup(test); - INT_TYPE = getEntityByName("type.basic.integer", "", ADMIN_AUTH_HEADERS); + INT_TYPE = getEntityByName("integer", "", ADMIN_AUTH_HEADERS); + } + + @Override + @Test + public void post_entityCreateWithInvalidName_400() { + String[][] tests = { + {"Abcd", "[name must match \"^[a-z][a-zA-Z0-9]+$\"]"}, + {"a bc", "[name must match \"^[a-z][a-zA-Z0-9]+$\"]"}, // Name must not have space + {"a_bc", "[name must match \"^[a-z][a-zA-Z0-9]+$\"]"}, // Name must not be underscored + {"a-bc", "[name must match \"^[a-z][a-zA-Z0-9]+$\"]"}, // Name must not be hyphened + }; + + CreateType create = createRequest("placeHolder", "", "", null); + for (String[] test : tests) { + LOG.info("Testing with the name {}", test[0]); + create.withName(test[0]); + assertResponseContains(() -> createEntity(create, ADMIN_AUTH_HEADERS), Status.BAD_REQUEST, test[1]); + } + } + + @Test + public void put_customField_200() throws HttpResponseException { + Type tableEntity = getEntityByName("table", "customFields", ADMIN_AUTH_HEADERS); + assertTrue(listOrEmpty(tableEntity.getCustomFields()).isEmpty()); + + // Add a custom field with name intA with type integer + CustomField fieldA = + new CustomField().withName("intA").withDescription("intA").withFieldType(INT_TYPE.getEntityReference()); + tableEntity = addCustomField(tableEntity.getId(), fieldA, Status.OK, ADMIN_AUTH_HEADERS); + assertEquals(1, tableEntity.getCustomFields().size()); + assertEquals(fieldA, tableEntity.getCustomFields().get(0)); + + // Add a second field with name intB with type integer + CustomField fieldB = + new CustomField().withName("intB").withDescription("intB").withFieldType(INT_TYPE.getEntityReference()); + tableEntity = addCustomField(tableEntity.getId(), fieldB, Status.OK, ADMIN_AUTH_HEADERS); + assertEquals(2, tableEntity.getCustomFields().size()); + assertEquals(fieldA, tableEntity.getCustomFields().get(0)); + assertEquals(fieldB, tableEntity.getCustomFields().get(1)); + } + + @Test + public void put_customFieldToFieldType_4xx() { + // Adding a custom field to a field type is not allowed (only entity type is allowed) + CustomField field = + new CustomField().withName("intA").withDescription("intA").withFieldType(INT_TYPE.getEntityReference()); + assertResponse( + () -> addCustomField(INT_TYPE.getId(), field, Status.CREATED, ADMIN_AUTH_HEADERS), + Status.BAD_REQUEST, + "Field types can't be extended"); } @Override @@ -61,12 +122,23 @@ public class TypeResourceTest extends EntityResourceTest { return type; } + public Type addCustomField(UUID entityTypeId, CustomField customField, Status status, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource(entityTypeId); + return TestUtils.put(target, customField, Type.class, status, authHeaders); + } + @Override public CreateType createRequest(String name, String description, String displayName, EntityReference owner) { + if (name != null) { + name = name.replaceAll("[. _-]", ""); + } + System.out.println("XXX Using the name " + name); return new CreateType() .withName(name) .withDescription(description) .withDisplayName(displayName) + .withCategory(Category.Field) .withSchema(INT_TYPE.getSchema()); }