Fixes #4977 Add APIs for adding custom fields to an existing entity (#4978)

bootstrap/sql/com.mysql.cj.jdbc.Driver/v002__create_db_connection_info.sql

@@ -1,18 +1,12 @@
---
+CREATE TABLE IF NOT EXISTS type_entity (
--- Table to be used for generic entities that are limited in number. Examples of generic entities are
--- Attribute entity, domain entities etc.
---
--- This reduces need for defining a table per entity.
---
-CREATE TABLE IF NOT EXISTS generic_entity (
     id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
-    -- Fully qualified name formed by entityType + "." + entityName
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
-    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    category VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.category') NOT NULL,
     json JSON NOT NULL,
     updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
     updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
     PRIMARY KEY (id),
-    UNIQUE (fullyQualifiedName)
+    UNIQUE (name)
 );
 
 ALTER TABLE webhook_entity

bootstrap/sql/org.postgresql.Driver/v002__create_db_connection_info.sql

@@ -1,18 +1,12 @@
---
+CREATE TABLE IF NOT EXISTS type_entity (
--- Table to be used for generic entities that are limited in number. Examples of generic entities are
--- Attribute entity, domain entities etc.
---
--- This reduces need for defining a table per entity.
---
-CREATE TABLE IF NOT EXISTS generic_entity (
     id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
-    -- Fully qualified name formed by entityType + "." + entityName
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
-    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    category VARCHAR(256) GENERATED ALWAYS AS (json ->> 'category') STORED NOT NULL,
     json JSONB NOT NULL,
     updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
     updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
     PRIMARY KEY (id),
-    UNIQUE (fullyQualifiedName)
+    UNIQUE (name)
 );
 
 ALTER TABLE webhook_entity

catalog-rest-service/pom.xml

@@ -486,6 +486,9 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <version>${maven-surefire.version}</version>
+        <configuration>
+          <trimStackTrace>false</trimStackTrace>
+        </configuration>
         <executions>
           <execution>
             <id>default-test</id>

catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogGenericExceptionMapper.java

@@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory;
 public class CatalogGenericExceptionMapper implements ExceptionMapper<Throwable> {
   @Override
   public Response toResponse(Throwable ex) {
-    ex.printStackTrace();
     LOG.debug(ex.getMessage());
     if (ex instanceof ProcessingException || ex instanceof IllegalArgumentException) {
       final Response response = BadRequestException.of().getResponse();

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java

@@ -176,7 +176,7 @@ public interface CollectionDAO {
   WebhookDAO webhookDAO();
 
   @CreateSqlObject
-  GenericEntityDAO genericEntityDAO();
+  TypeEntityDAO typeEntityDAO();
 
   interface DashboardDAO extends EntityDAO<Dashboard> {
     @Override
@@ -339,16 +339,23 @@ public interface CollectionDAO {
 
   interface EntityRelationshipDAO {
     default int insert(UUID fromId, UUID toId, String fromEntity, String toEntity, int relation) {
-      return insert(fromId.toString(), toId.toString(), fromEntity, toEntity, relation);
+      return insert(fromId, toId, fromEntity, toEntity, relation, null);
+    }
+
+    default int insert(UUID fromId, UUID toId, String fromEntity, String toEntity, int relation, String json) {
+      return insert(fromId.toString(), toId.toString(), fromEntity, toEntity, relation, json);
     }
 
     @ConnectionAwareSqlUpdate(
         value =
-            "INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation)",
+            "INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) "
+                + "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation)",
         connectionType = MYSQL)
     @ConnectionAwareSqlUpdate(
         value =
-            "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation) ON CONFLICT (fromId, toId, relation) DO NOTHING",
+            "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) "
+                + "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation) "
+                + "ON CONFLICT (fromId, toId, relation) DO NOTHING",
         connectionType = POSTGRES)
     int insert(
         @Bind("fromId") String fromId,
@@ -357,6 +364,25 @@ public interface CollectionDAO {
         @Bind("toEntity") String toEntity,
         @Bind("relation") int relation);
 
+    @ConnectionAwareSqlUpdate(
+        value =
+            "INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation, json) "
+                + "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation, :json)",
+        connectionType = MYSQL)
+    @ConnectionAwareSqlUpdate(
+        value =
+            "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation, json) VALUES "
+                + "(:fromId, :toId, :fromEntity, :toEntity, :relation, (:json :: jsonb)) "
+                + "ON CONFLICT (fromId, toId, relation) DO NOTHING",
+        connectionType = POSTGRES)
+    int insert(
+        @Bind("fromId") String fromId,
+        @Bind("toId") String toId,
+        @Bind("fromEntity") String fromEntity,
+        @Bind("toEntity") String toEntity,
+        @Bind("relation") int relation,
+        @Bind("json") String json);
+
     //
     // Find to operations
     //
@@ -368,17 +394,6 @@ public interface CollectionDAO {
     List<EntityReference> findTo(
         @Bind("fromId") String fromId, @Bind("fromEntity") String fromEntity, @Bind("relation") int relation);
 
-    @SqlQuery(
-        "SELECT toId, toEntity FROM entity_relationship "
-            + "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity "
-            + "ORDER BY toId")
-    @RegisterRowMapper(ToEntityReferenceMapper.class)
-    List<EntityReference> findToReference(
-        @Bind("fromId") String fromId,
-        @Bind("fromEntity") String fromEntity,
-        @Bind("relation") int relation,
-        @Bind("toEntity") String toEntity);
-
     @SqlQuery(
         "SELECT toId FROM entity_relationship "
             + "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity "
@@ -421,17 +436,6 @@ public interface CollectionDAO {
     List<EntityReference> findFrom(
         @Bind("toId") String toId, @Bind("toEntity") String toEntity, @Bind("relation") int relation);
 
-    @SqlQuery(
-        "SELECT fromId, fromEntity FROM entity_relationship "
-            + "WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation AND fromEntity = :fromEntity "
-            + "ORDER BY fromId")
-    @RegisterRowMapper(FromEntityReferenceMapper.class)
-    List<EntityReference> findFromEntity(
-        @Bind("toId") String toId,
-        @Bind("toEntity") String toEntity,
-        @Bind("relation") int relation,
-        @Bind("fromEntity") String fromEntity);
-
     //
     // Delete Operations
     //
@@ -709,18 +713,22 @@ public interface CollectionDAO {
   interface FieldRelationshipDAO {
     @ConnectionAwareSqlUpdate(
         value =
-            "INSERT IGNORE INTO field_relationship(fromFQN, toFQN, fromType, toType, relation) VALUES (:fromFQN, :toFQN, :fromType, :toType, :relation)",
+            "INSERT IGNORE INTO field_relationship(fromFQN, toFQN, fromType, toType, relation, json) "
+                + "VALUES (:fromFQN, :toFQN, :fromType, :toType, :relation, :json)",
         connectionType = MYSQL)
     @ConnectionAwareSqlUpdate(
         value =
-            "INSERT INTO field_relationship(fromFQN, toFQN, fromType, toType, relation) VALUES (:fromFQN, :toFQN, :fromType, :toType, :relation) ON CONFLICT (fromFQN, toFQN, relation) DO NOTHING",
+            "INSERT INTO field_relationship(fromFQN, toFQN, fromType, toType, relation, json) "
+                + "VALUES (:fromFQN, :toFQN, :fromType, :toType, :relation, (:json :: jsonb)) "
+                + "ON CONFLICT (fromFQN, toFQN, relation) DO NOTHING",
         connectionType = POSTGRES)
-    void insert(
+    int insert(
         @Bind("fromFQN") String fromFQN,
         @Bind("toFQN") String toFQN,
         @Bind("fromType") String fromType,
         @Bind("toType") String toType,
-        @Bind("relation") int relation);
+        @Bind("relation") int relation,
+        @Bind("json") String json);
 
     @ConnectionAwareSqlUpdate(
         value =
@@ -794,6 +802,16 @@ public interface CollectionDAO {
     @SqlUpdate("DELETE from field_relationship <cond>")
     void deleteAllByPrefixInternal(@Define("cond") String cond);
 
+    @SqlUpdate(
+        "DELETE from field_relationship WHERE fromFQN = :fromFQN AND toFQN = :toFQN AND fromType = :fromType "
+            + "AND toType = :toType AND relation = :relation")
+    void delete(
+        @Bind("fromFQN") String fromFQN,
+        @Bind("toFQN") String toFQN,
+        @Bind("fromType") String fromType,
+        @Bind("toType") String toType,
+        @Bind("relation") int relation);
+
     class ToFieldMapper implements RowMapper<List<String>> {
       @Override
       public List<String> map(ResultSet rs, StatementContext ctx) throws SQLException {
@@ -1560,10 +1578,10 @@ public interface CollectionDAO {
     List<String> listWithoutEntityFilter(@Bind("eventType") String eventType, @Bind("timestamp") long timestamp);
   }
 
-  interface GenericEntityDAO extends EntityDAO<Type> {
+  interface TypeEntityDAO extends EntityDAO<Type> {
     @Override
     default String getTableName() {
-      return "generic_entity";
+      return "type_entity";
     }
 
     @Override
@@ -1573,7 +1591,7 @@ public interface CollectionDAO {
 
     @Override
     default String getNameColumn() {
-      return "fullyQualifiedName";
+      return "name";
     }
 
     @Override

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java

@@ -115,7 +115,7 @@ import org.openmetadata.catalog.util.ResultList;
 public abstract class EntityRepository<T extends EntityInterface> {
   private final String collectionPath;
   private final Class<T> entityClass;
-  private final String entityType;
+  protected final String entityType;
   public final EntityDAO<T> dao;
   protected final CollectionDAO daoCollection;
   protected final List<String> allowedFields;
@@ -129,7 +129,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
   private final Fields patchFields;
 
   /** Fields that can be updated during PUT operation */
-  private final Fields putFields;
+  protected final Fields putFields;
 
   EntityRepository(
       String collectionPath,
@@ -737,8 +737,24 @@ public abstract class EntityRepository<T extends EntityInterface> {
     return addRelationship(fromId, toId, fromEntity, toEntity, relationship, false);
   }
 
+  public int addRelationship(
+      UUID fromId, UUID toId, String fromEntity, String toEntity, Relationship relationship, String json) {
+    return addRelationship(fromId, toId, fromEntity, toEntity, relationship, json, false);
+  }
+
   public int addRelationship(
       UUID fromId, UUID toId, String fromEntity, String toEntity, Relationship relationship, boolean bidirectional) {
+    return addRelationship(fromId, toId, fromEntity, toEntity, relationship, null, bidirectional);
+  }
+
+  public int addRelationship(
+      UUID fromId,
+      UUID toId,
+      String fromEntity,
+      String toEntity,
+      Relationship relationship,
+      String json,
+      boolean bidirectional) {
     UUID from = fromId;
     UUID to = toId;
     if (bidirectional && fromId.compareTo(toId) > 0) {
@@ -747,7 +763,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
       from = toId;
       to = fromId;
     }
-    return daoCollection.relationshipDAO().insert(from, to, fromEntity, toEntity, relationship.ordinal());
+    return daoCollection.relationshipDAO().insert(fromId, toId, fromEntity, toEntity, relationship.ordinal(), json);
   }
 
   public List<String> findBoth(UUID entity1, String entityType1, Relationship relationship, String entity2) {

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/FeedRepository.java

@@ -13,6 +13,11 @@
 
 package org.openmetadata.catalog.jdbi3;
 
+import static org.openmetadata.catalog.type.Relationship.ADDRESSED_TO;
+import static org.openmetadata.catalog.type.Relationship.CREATED;
+import static org.openmetadata.catalog.type.Relationship.IS_ABOUT;
+import static org.openmetadata.catalog.type.Relationship.REPLIED_TO;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -83,13 +88,7 @@ public class FeedRepository {
     dao.feedDAO().insert(JsonUtils.pojoToJson(thread));
 
     // Add relationship User -- created --> Thread relationship
-    dao.relationshipDAO()
+    dao.relationshipDAO().insert(createdByUser.getId(), thread.getId(), Entity.USER, Entity.THREAD, CREATED.ordinal());
-        .insert(
-            createdByUser.getId().toString(),
-            thread.getId().toString(),
-            Entity.USER,
-            Entity.THREAD,
-            Relationship.CREATED.ordinal());
 
     // Add field relationship data asset Thread -- isAbout ---> entity/entityField
     // relationship
@@ -99,17 +98,13 @@ public class FeedRepository {
             about.getFullyQualifiedFieldValue(), // to FQN
             Entity.THREAD, // From type
             about.getFullyQualifiedFieldType(), // to Type
-            Relationship.IS_ABOUT.ordinal());
+            IS_ABOUT.ordinal(),
+            null);
 
     // Add the owner also as addressedTo as the entity he owns when addressed, the owner is actually being addressed
     if (entityOwner != null) {
       dao.relationshipDAO()
-          .insert(
+          .insert(thread.getId(), entityOwner.getId(), Entity.THREAD, entityOwner.getType(), ADDRESSED_TO.ordinal());
-              thread.getId().toString(),
-              entityOwner.getId().toString(),
-              Entity.THREAD,
-              entityOwner.getType(),
-              Relationship.ADDRESSED_TO.ordinal());
     }
 
     // Add mentions to field relationship table
@@ -151,7 +146,8 @@ public class FeedRepository {
                         thread.getId().toString(),
                         mention.getFullyQualifiedFieldType(),
                         Entity.THREAD,
-                        Relationship.MENTIONED_IN.ordinal()));
+                        Relationship.MENTIONED_IN.ordinal(),
+                        null));
   }
 
   @Transaction
@@ -178,13 +174,7 @@ public class FeedRepository {
       }
     }
     if (!relationAlreadyExists) {
-      dao.relationshipDAO()
+      dao.relationshipDAO().insert(fromUser.getId(), thread.getId(), Entity.USER, Entity.THREAD, REPLIED_TO.ordinal());
-          .insert(
-              fromUser.getId().toString(),
-              thread.getId().toString(),
-              Entity.USER,
-              Entity.THREAD,
-              Relationship.REPLIED_TO.ordinal());
     }
 
     // Add mentions into field relationship table
@@ -233,7 +223,7 @@ public class FeedRepository {
       result =
           dao.feedDAO()
               .listCountByEntityLink(
-                  StringUtils.EMPTY, Entity.THREAD, StringUtils.EMPTY, Relationship.IS_ABOUT.ordinal(), isResolved);
+                  StringUtils.EMPTY, Entity.THREAD, StringUtils.EMPTY, IS_ABOUT.ordinal(), isResolved);
     } else {
       EntityLink entityLink = EntityLink.parse(link);
       EntityReference reference = EntityUtil.validateEntityLink(entityLink);
@@ -253,7 +243,7 @@ public class FeedRepository {
                     entityLink.getFullyQualifiedFieldValue(),
                     Entity.THREAD,
                     entityLink.getFullyQualifiedFieldType(),
-                    Relationship.IS_ABOUT.ordinal(),
+                    IS_ABOUT.ordinal(),
                     isResolved);
       }
     }
@@ -333,7 +323,7 @@ public class FeedRepository {
                         limit + 1,
                         time,
                         isResolved,
-                        Relationship.IS_ABOUT.ordinal());
+                        IS_ABOUT.ordinal());
           } else {
             jsons =
                 dao.feedDAO()
@@ -343,7 +333,7 @@ public class FeedRepository {
                         limit + 1,
                         time,
                         isResolved,
-                        Relationship.IS_ABOUT.ordinal());
+                        IS_ABOUT.ordinal());
           }
           threads = JsonUtils.readObjects(jsons, Thread.class);
           total =
@@ -352,7 +342,7 @@ public class FeedRepository {
                       entityLink.getFullyQualifiedFieldValue(),
                       entityLink.getFullyQualifiedFieldType(),
                       isResolved,
-                      Relationship.IS_ABOUT.ordinal());
+                      IS_ABOUT.ordinal());
         }
       } else {
         FilteredThreads filteredThreads;
@@ -474,7 +464,7 @@ public class FeedRepository {
       String userId, int limit, long time, boolean isResolved, PaginationType paginationType) throws IOException {
     List<EntityReference> teams =
         EntityUtil.populateEntityReferences(
-            dao.relationshipDAO().findFromEntity(userId, Entity.USER, Relationship.HAS.ordinal(), Entity.TEAM));
+            dao.relationshipDAO().findFrom(userId, Entity.USER, Relationship.HAS.ordinal(), Entity.TEAM), Entity.TEAM);
     List<String> teamNames = teams.stream().map(EntityReference::getName).collect(Collectors.toList());
     if (teamNames.isEmpty()) {
       teamNames = List.of(StringUtils.EMPTY);

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/RoleRepository.java

@@ -25,9 +25,6 @@ import lombok.extern.slf4j.Slf4j;
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.entity.teams.Role;
-import org.openmetadata.catalog.entity.teams.User;
-import org.openmetadata.catalog.exception.CatalogExceptionMessage;
-import org.openmetadata.catalog.exception.EntityNotFoundException;
 import org.openmetadata.catalog.resources.teams.RoleResource;
 import org.openmetadata.catalog.type.EntityReference;
 import org.openmetadata.catalog.type.Relationship;
@@ -178,18 +175,5 @@ public class RoleRepository extends EntityRepository<Role> {
           .relationshipDAO()
           .deleteTo(role.getId().toString(), Entity.ROLE, Relationship.HAS.ordinal(), Entity.USER);
     }
-
-    private List<User> getAllUsers() {
-      EntityRepository<User> userRepository = Entity.getEntityRepository(Entity.USER);
-      try {
-        // Assumptions:
-        // - we will not have more than Integer.MAX_VALUE users in the system.
-        // - we do not need to update deleted user's roles.
-        ListFilter filter = new ListFilter();
-        return userRepository.listAfter(null, Fields.EMPTY_FIELDS, filter, Integer.MAX_VALUE - 1, null).getData();
-      } catch (IOException e) {
-        throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entitiesNotFound(Entity.USER));
-      }
-    }
   }
 }

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TypeRepository.java

@@ -16,34 +16,43 @@
 
 package org.openmetadata.catalog.jdbi3;
 
+import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
 import java.io.IOException;
 import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import javax.ws.rs.core.UriInfo;
+import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.entity.Type;
+import org.openmetadata.catalog.entity.type.Category;
+import org.openmetadata.catalog.entity.type.CustomField;
 import org.openmetadata.catalog.resources.types.TypeResource;
+import org.openmetadata.catalog.type.Include;
+import org.openmetadata.catalog.type.Relationship;
+import org.openmetadata.catalog.util.EntityUtil;
 import org.openmetadata.catalog.util.EntityUtil.Fields;
 import org.openmetadata.catalog.util.FullyQualifiedName;
+import org.openmetadata.catalog.util.JsonUtils;
+import org.openmetadata.catalog.util.RestUtil.PutResponse;
 
+@Slf4j
 public class TypeRepository extends EntityRepository<Type> {
-  // TODO fix this
+  private static final String UPDATE_FIELDS = "customFields";
-  private static final String UPDATE_FIELDS = "";
+  private static final String PATCH_FIELDS = "customFields";
-  private static final String PATCH_FIELDS = "";
 
   public TypeRepository(CollectionDAO dao) {
-    super(
+    super(TypeResource.COLLECTION_PATH, Entity.TYPE, Type.class, dao.typeEntityDAO(), dao, PATCH_FIELDS, UPDATE_FIELDS);
-        TypeResource.COLLECTION_PATH,
-        Entity.TYPE,
-        Type.class,
-        dao.genericEntityDAO(),
-        dao,
-        PATCH_FIELDS,
-        UPDATE_FIELDS);
     allowEdits = true;
   }
 
   @Override
-  public Type setFields(Type attribute, Fields fields) throws IOException {
+  public Type setFields(Type type, Fields fields) throws IOException {
-    return attribute;
+    type.withCustomFields(fields.contains("customFields") ? getCustomFields(type) : null);
+    return type;
   }
 
   @Override
@@ -54,9 +63,10 @@ public class TypeRepository extends EntityRepository<Type> {
   @Override
   public void storeEntity(Type type, boolean update) throws IOException {
     URI href = type.getHref();
-    type.withHref(null);
+    List<CustomField> customFields = type.getCustomFields();
+    type.withHref(null).withCustomFields(null);
     store(type.getId(), type, update);
-    type.withHref(href);
+    type.withHref(href).withCustomFields(customFields);
   }
 
   @Override
@@ -65,19 +75,95 @@ public class TypeRepository extends EntityRepository<Type> {
   }
 
   @Override
-  public void setFullyQualifiedName(Type entity) {
+  public EntityUpdater getUpdater(Type original, Type updated, Operation operation) {
-    entity.setFullyQualifiedName(FullyQualifiedName.build(Entity.TYPE, entity.getNameSpace(), entity.getName()));
+    return new TypeUpdater(original, updated, operation);
   }
 
-  @Override
+  public PutResponse<Type> addCustomField(UriInfo uriInfo, String updatedBy, String id, CustomField field)
-  public EntityUpdater getUpdater(Type original, Type updated, Operation operation) {
+      throws IOException {
-    return new AttributeUpdater(original, updated, operation);
+    Type type = dao.findEntityById(UUID.fromString(id), Include.NON_DELETED);
+    if (type.getCategory().equals(Category.Field)) {
+      throw new IllegalArgumentException("Field types can't be extended");
+    }
+    setFields(type, putFields);
+
+    dao.findEntityById(field.getFieldType().getId()); // Validate customField type exists
+    type.getCustomFields().add(field);
+    type.setUpdatedBy(updatedBy);
+    type.setUpdatedAt(System.currentTimeMillis());
+    return createOrUpdate(uriInfo, type);
+  }
+
+  private String getCustomFieldFQNPrefix(Type type) {
+    return FullyQualifiedName.build(type.getName(), "customFields");
+  }
+
+  private String getCustomFieldFQN(Type type, String fieldName) {
+    return FullyQualifiedName.build(type.getName(), "customFields", fieldName);
+  }
+
+  private List<CustomField> getCustomFields(Type type) throws IOException {
+    List<CustomField> customFields = new ArrayList<>();
+    List<List<String>> results =
+        daoCollection
+            .fieldRelationshipDAO()
+            .listToByPrefix(getCustomFieldFQNPrefix(type), Entity.TYPE, Entity.TYPE, Relationship.HAS.ordinal());
+    for (List<String> result : results) {
+      CustomField field = JsonUtils.readValue(result.get(2), CustomField.class);
+      field.setFieldType(dao.findEntityReferenceByName(result.get(1)));
+      customFields.add(field);
+    }
+    customFields.sort(EntityUtil.compareCustomField);
+    return customFields;
   }
 
   /** Handles entity updated from PUT and POST operation. */
-  public class AttributeUpdater extends EntityUpdater {
+  public class TypeUpdater extends EntityUpdater {
-    public AttributeUpdater(Type original, Type updated, Operation operation) {
+    public TypeUpdater(Type original, Type updated, Operation operation) {
       super(original, updated, operation);
     }
+
+    @Override
+    public void entitySpecificUpdate() throws IOException {
+      updateCustomFields();
+    }
+
+    private void updateCustomFields() throws JsonProcessingException {
+      List<CustomField> updatedFields = listOrEmpty(updated.getCustomFields());
+      List<CustomField> origFields = listOrEmpty(original.getCustomFields());
+      List<CustomField> added = new ArrayList<>();
+      List<CustomField> deleted = new ArrayList<>();
+      recordListChange("charts", origFields, updatedFields, added, deleted, EntityUtil.customFieldMatch);
+      for (CustomField field : added) {
+        String customFieldFQN = getCustomFieldFQN(updated, field.getName());
+        String customFieldJson = JsonUtils.pojoToJson(field);
+        LOG.info(
+            "Adding customField {} with type {} to the entity {}",
+            field.getName(),
+            field.getFieldType().getName(),
+            updated.getName());
+        daoCollection
+            .fieldRelationshipDAO()
+            .insert(
+                customFieldFQN,
+                field.getFieldType().getName(),
+                Entity.TYPE,
+                Entity.TYPE,
+                Relationship.HAS.ordinal(),
+                customFieldJson);
+      }
+      for (CustomField field : deleted) {
+        String customFieldFQN = getCustomFieldFQN(updated, field.getName());
+        LOG.info(
+            "Deleting customField {} with type {} from the entity {}",
+            field.getName(),
+            field.getFieldType().getName(),
+            updated.getName());
+        daoCollection
+            .fieldRelationshipDAO()
+            .delete(
+                customFieldFQN, field.getFieldType().getName(), Entity.TYPE, Entity.TYPE, Relationship.HAS.ordinal());
+      }
+    }
   }
 }

catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/types/TypeResource.java

@@ -16,6 +16,7 @@ package org.openmetadata.catalog.resources.types;
 import static org.openmetadata.catalog.security.SecurityUtil.ADMIN;
 import static org.openmetadata.catalog.security.SecurityUtil.BOT;
 import static org.openmetadata.catalog.security.SecurityUtil.OWNER;
+import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
 
 import com.google.inject.Inject;
 import io.swagger.annotations.Api;
@@ -52,18 +53,22 @@ import javax.ws.rs.core.SecurityContext;
 import javax.ws.rs.core.UriInfo;
 import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.catalog.CatalogApplicationConfig;
+import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.api.CreateType;
 import org.openmetadata.catalog.entity.Type;
+import org.openmetadata.catalog.entity.type.Category;
+import org.openmetadata.catalog.entity.type.CustomField;
 import org.openmetadata.catalog.jdbi3.CollectionDAO;
 import org.openmetadata.catalog.jdbi3.ListFilter;
 import org.openmetadata.catalog.jdbi3.TypeRepository;
 import org.openmetadata.catalog.resources.Collection;
 import org.openmetadata.catalog.resources.EntityResource;
 import org.openmetadata.catalog.security.Authorizer;
+import org.openmetadata.catalog.security.SecurityUtil;
 import org.openmetadata.catalog.type.EntityHistory;
 import org.openmetadata.catalog.type.Include;
-import org.openmetadata.catalog.util.EntityUtil;
 import org.openmetadata.catalog.util.JsonUtils;
+import org.openmetadata.catalog.util.RestUtil.PutResponse;
 import org.openmetadata.catalog.util.ResultList;
 
 @Path("/v1/metadata/types")
@@ -77,7 +82,8 @@ public class TypeResource extends EntityResource<Type, TypeRepository> {
 
   @Override
   public Type addHref(UriInfo uriInfo, Type type) {
-    return type; // Nothing to do
+    listOrEmpty(type.getCustomFields()).forEach(field -> Entity.withHref(uriInfo, field.getFieldType()));
+    return type;
   }
 
   @Inject
@@ -88,26 +94,18 @@ public class TypeResource extends EntityResource<Type, TypeRepository> {
   @SuppressWarnings("unused") // Method used for reflection
   public void initialize(CatalogApplicationConfig config) throws IOException {
     // Find tag definitions and load tag categories from the json file, if necessary
-    List<String> jsonSchemas = EntityUtil.getJsonDataResources(".*json/schema/type/.*\\.json$");
     long now = System.currentTimeMillis();
-    for (String jsonSchema : jsonSchemas) {
+    List<Type> types = JsonUtils.getTypes();
-      try {
+    types.forEach(
-        List<Type> types = JsonUtils.getTypes(jsonSchema);
+        type -> {
-        types.forEach(
+          type.withId(UUID.randomUUID()).withUpdatedBy("admin").withUpdatedAt(now);
-            type -> {
+          LOG.info("Loading type {} with schema {}", type.getName(), type.getSchema());
-              type.withId(UUID.randomUUID()).withUpdatedBy("admin").withUpdatedAt(now);
+          try {
-              LOG.info("Loading from {} type {} with schema {}", jsonSchema, type.getName(), type.getSchema());
+            this.dao.createOrUpdate(null, type);
-              try {
+          } catch (IOException e) {
-                this.dao.createOrUpdate(null, type);
+            LOG.error("Error loading type {}", type.getName(), e);
-              } catch (IOException e) {
+          }
-                LOG.error("Error loading type {} from {}", type.getName(), jsonSchema, e);
+        });
-                e.printStackTrace();
-              }
-            });
-      } catch (Exception e) {
-        LOG.warn("Failed to initialize the types from jsonSchema file {}", jsonSchema, e);
-      }
-    }
   }
 
   public static class TypeList extends ResultList<Type> {
@@ -121,7 +119,7 @@ public class TypeResource extends EntityResource<Type, TypeRepository> {
     }
   }
 
-  public static final String FIELDS = "";
+  public static final String FIELDS = "customFields";
 
   @GET
   @Valid
@@ -141,6 +139,11 @@ public class TypeResource extends EntityResource<Type, TypeRepository> {
   public ResultList<Type> list(
       @Context UriInfo uriInfo,
       @Context SecurityContext securityContext,
+      @Parameter(
+              description = "Filter types by metadata type category.",
+              schema = @Schema(type = "string", example = "Field, Entity"))
+          @QueryParam("category")
+          Category category,
       @Parameter(description = "Limit the number types returned. (1 to 1000000, " + "default = 10)")
           @DefaultValue("10")
           @Min(0)
@@ -349,11 +352,35 @@ public class TypeResource extends EntityResource<Type, TypeRepository> {
     return delete(uriInfo, securityContext, id, false, true, ADMIN | BOT);
   }
 
+  @PUT
+  @Path("/{id}")
+  @Operation(
+      summary = "Add a field to an entity",
+      tags = "metadata",
+      description = "Add a field to an entity type. Fields can only be added to entity type and not field type.",
+      responses = {
+        @ApiResponse(responseCode = "200", description = "OK"),
+        @ApiResponse(responseCode = "404", description = "type for instance {id} is not found")
+      })
+  public Response addField(
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext,
+      @Parameter(description = "Type Id", schema = @Schema(type = "string")) @PathParam("id") String id,
+      CustomField field)
+      throws IOException {
+    SecurityUtil.authorizeAdmin(authorizer, securityContext, ADMIN | BOT);
+    PutResponse<Type> response = dao.addCustomField(uriInfo, securityContext.getUserPrincipal().getName(), id, field);
+    addHref(uriInfo, response.getEntity());
+    return response.toResponse();
+  }
+
   private Type getType(SecurityContext securityContext, CreateType create) {
     return new Type()
         .withId(UUID.randomUUID())
         .withName(create.getName())
+        .withFullyQualifiedName(create.getName())
         .withDisplayName(create.getDisplayName())
+        .withCategory(create.getCategory())
         .withSchema(create.getSchema())
         .withDescription(create.getDescription())
         .withUpdatedBy(securityContext.getUserPrincipal().getName())

catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java

@@ -40,6 +40,7 @@ import org.openmetadata.catalog.EntityInterface;
 import org.openmetadata.catalog.api.data.TermReference;
 import org.openmetadata.catalog.entity.data.GlossaryTerm;
 import org.openmetadata.catalog.entity.data.Table;
+import org.openmetadata.catalog.entity.type.CustomField;
 import org.openmetadata.catalog.exception.CatalogExceptionMessage;
 import org.openmetadata.catalog.exception.EntityNotFoundException;
 import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityVersionPair;
@@ -80,6 +81,7 @@ public final class EntityUtil {
       Comparator.comparing(TableConstraint::getConstraintType);
   public static final Comparator<ChangeEvent> compareChangeEvent = Comparator.comparing(ChangeEvent::getTimestamp);
   public static final Comparator<GlossaryTerm> compareGlossaryTerm = Comparator.comparing(GlossaryTerm::getName);
+  public static final Comparator<CustomField> compareCustomField = Comparator.comparing(CustomField::getName);
 
   //
   // Matchers used for matching two items in a list
@@ -130,6 +132,9 @@ public final class EntityUtil {
   public static final BiPredicate<TermReference, TermReference> termReferenceMatch =
       (ref1, ref2) -> ref1.getName().equals(ref2.getName()) && ref1.getEndpoint().equals(ref2.getEndpoint());
 
+  public static final BiPredicate<CustomField, CustomField> customFieldMatch =
+      (ref1, ref2) -> ref1.getName().equals(ref2.getName());
+
   private EntityUtil() {}
 
   /** Validate Ingestion Schedule */

catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonUtils.java

@@ -45,10 +45,14 @@ import javax.json.JsonReader;
 import javax.json.JsonStructure;
 import javax.json.JsonValue;
 import javax.ws.rs.core.MediaType;
+import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.catalog.entity.Type;
+import org.openmetadata.catalog.entity.type.Category;
 
+@Slf4j
 public final class JsonUtils {
-  public static final String TYPE_ANNOTATION = "@om-type";
+  public static final String FIELD_TYPE_ANNOTATION = "@om-field-type";
+  public static final String ENTITY_TYPE_ANNOTATION = "@om-entity-type";
   public static final MediaType DEFAULT_MEDIA_TYPE = MediaType.APPLICATION_JSON_TYPE;
   private static final ObjectMapper OBJECT_MAPPER;
   private static final JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(VersionFlag.V7);
@@ -217,11 +221,39 @@ public final class JsonUtils {
     return comment != null && comment.contains(annotation);
   }
 
+  /** Get all the fields types and entity types from OpenMetadata JSON schema definition files. */
+  public static List<Type> getTypes() throws IOException {
+    // Get Field Types
+    List<Type> types = new ArrayList<>();
+    List<String> jsonSchemas = EntityUtil.getJsonDataResources(".*json/schema/type/.*\\.json$");
+    for (String jsonSchema : jsonSchemas) {
+      try {
+        types.addAll(JsonUtils.getFieldTypes(jsonSchema));
+      } catch (Exception e) {
+        LOG.warn("Failed to initialize the types from jsonSchema file {}", jsonSchema, e);
+      }
+    }
+
+    // Get Entity Types
+    jsonSchemas = EntityUtil.getJsonDataResources(".*json/schema/entity/.*\\.json$");
+    for (String jsonSchema : jsonSchemas) {
+      try {
+        Type entityType = JsonUtils.getEntityType(jsonSchema);
+        if (entityType != null) {
+          types.add(entityType);
+        }
+      } catch (Exception e) {
+        LOG.warn("Failed to initialize the types from jsonSchema file {}", jsonSchema, e);
+      }
+    }
+    return types;
+  }
+
   /**
-   * Get all the types from the `definitions` section of a JSON schema file that are annotated with "$comment" field set
+   * Get all the fields types from the `definitions` section of a JSON schema file that are annotated with "$comment"
-   * to "@om-type".
+   * field set to "@om-field-type".
    */
-  public static List<Type> getTypes(String jsonSchemaFile) throws IOException {
+  public static List<Type> getFieldTypes(String jsonSchemaFile) throws IOException {
     JsonNode node =
         OBJECT_MAPPER.readTree(
             Objects.requireNonNull(JsonUtils.class.getClassLoader().getResourceAsStream(jsonSchemaFile)));
@@ -236,12 +268,15 @@ public final class JsonUtils {
     Iterator<Entry<String, JsonNode>> definitions = node.get("definitions").fields();
     while (definitions != null && definitions.hasNext()) {
       Entry<String, JsonNode> entry = definitions.next();
+      String typeName = entry.getKey();
       JsonNode value = entry.getValue();
-      if (JsonUtils.hasAnnotation(value, JsonUtils.TYPE_ANNOTATION)) {
+      if (JsonUtils.hasAnnotation(value, JsonUtils.FIELD_TYPE_ANNOTATION)) {
         String description = String.valueOf(value.get("description"));
         Type type =
             new Type()
-                .withName(entry.getKey())
+                .withName(typeName)
+                .withCategory(Category.Field)
+                .withFullyQualifiedName(typeName)
                 .withNameSpace(jsonNamespace)
                 .withDescription(description)
                 .withDisplayName(entry.getKey())
@@ -251,4 +286,33 @@ public final class JsonUtils {
     }
     return types;
   }
+
+  /**
+   * Get all the fields types from the `definitions` section of a JSON schema file that are annotated with "$comment"
+   * field set to "@om-entity-type".
+   */
+  public static Type getEntityType(String jsonSchemaFile) throws IOException {
+    JsonNode node =
+        OBJECT_MAPPER.readTree(
+            Objects.requireNonNull(JsonUtils.class.getClassLoader().getResourceAsStream(jsonSchemaFile)));
+    if (!JsonUtils.hasAnnotation(node, JsonUtils.ENTITY_TYPE_ANNOTATION)) {
+      return null;
+    }
+
+    String fileName = Paths.get(jsonSchemaFile).getFileName().toString();
+    String entityName = fileName.replace(" ", "").replace(".json", "");
+
+    String namespaceFile = Paths.get(jsonSchemaFile).getParent().getFileName().toString();
+    String namespace = namespaceFile.replace(" ", "").replace(".json", "");
+
+    String description = String.valueOf(node.get("description"));
+    return new Type()
+        .withName(entityName)
+        .withCategory(Category.Entity)
+        .withFullyQualifiedName(entityName)
+        .withNameSpace(namespace)
+        .withDescription(description)
+        .withDisplayName(entityName)
+        .withSchema(node.toPrettyString());
+  }
 }

catalog-rest-service/src/main/resources/json/schema/api/createType.json

@@ -1,5 +1,5 @@
 {
-  "$id": "https://open-metadata.org/schema/api/data/createAttribute.json",
+  "$id": "https://open-metadata.org/schema/api/data/createType.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "createType",
   "description": "Create a Type to be used for extending entities.",
@@ -7,7 +7,7 @@
   "properties": {
     "name": {
       "description": "Unique name that identifies a Type.",
-      "$ref": "../type/basic.json#/definitions/entityName"
+      "$ref": "../entity/type.json#/definitions/typeName"
     },
     "nameSpace": {
       "description": "Namespace or group to which this type belongs to.",
@@ -20,7 +20,10 @@
     },
     "description": {
       "description": "Optional description of the type.",
-      "type": "string"
+      "$ref" : "../type/basic.json#/definitions/markdown"
+    },
+    "category" : {
+      "$ref" : "../entity/type.json#/definitions/category"
     },
     "schema": {
       "description": "JSON schema encoded as string. This will be used to validate the type values.",

catalog-rest-service/src/main/resources/json/schema/entity/data/table.json

@@ -2,6 +2,7 @@
   "$id": "https://open-metadata.org/schema/entity/data/table.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Table",
+  "$comment" : "@om-entity-type",
   "description": "This schema defines the Table entity. A Table organizes data in rows and columns and is defined by a Schema. OpenMetadata does not have a separate abstraction for Schema. Both Table and Schema are captured in this entity.",
   "type": "object",
   "javaType": "org.openmetadata.catalog.entity.data.Table",

catalog-rest-service/src/main/resources/json/schema/entity/type.json

@@ -2,10 +2,63 @@
   "$id": "https://open-metadata.org/schema/entity/type.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Type",
-  "description": "This schema defines a type entity used for extending an entity with custom attributes.",
+  "description": "This schema defines a type as an entity. Types includes field types and entity types. Custom types can also be defined by the users to extend the metadata system.",
   "type": "object",
   "javaType": "org.openmetadata.catalog.entity.Type",
   "javaInterfaces": ["org.openmetadata.catalog.EntityInterface"],
+  "definitions": {
+    "category" : {
+      "description" : "Metadata category to which a type belongs to.",
+      "type": "string",
+      "javaType": "org.openmetadata.catalog.entity.type.Category",
+      "enum": [
+        "field",
+        "entity"
+      ],
+      "javaEnums": [
+        {
+          "name": "Field"
+        },
+        {
+          "name": "Entity"
+        }
+      ]
+    },
+    "fieldName": {
+      "description": "Name of the entity field. Note a field name must be unique for an entity. Field name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.",
+      "type" : "string",
+      "pattern": "^[a-z][a-zA-Z0-9]+$"
+    },
+    "typeName": {
+      "description": "Name of the field or entity types. Note a field name must be unique for an entity. Field name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.",
+      "type" : "string",
+      "pattern": "^[a-z][a-zA-Z0-9]+$"
+    },
+    "customField" : {
+      "description" : "Type used for adding custom field to an entity to extend it.",
+      "type" : "object",
+      "javaType": "org.openmetadata.catalog.entity.type.CustomField",
+      "properties": {
+        "name": {
+          "description": "Name of the entity field. Note a field name must be unique for an entity. Field name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.",
+          "$ref" : "#/definitions/fieldName"
+        },
+        "description": {
+          "$ref" : "../type/basic.json#/definitions/markdown"
+        },
+        "fieldType": {
+          "description": "Reference to a field type. Only field types are allows and entity types are not allowed as custom fields to extend an existing entity",
+          "$ref": "../type/entityReference.json"
+        }
+      },
+      "required": [
+        "name",
+        "description",
+        "fieldType"
+      ],
+      "additionalProperties": false
+    }
+  },
   "properties": {
     "id": {
       "description": "Unique identifier of the type instance.",
@@ -13,29 +66,39 @@
     },
     "name": {
       "description": "Unique name that identifies the type.",
-      "$ref": "../type/basic.json#/definitions/entityName"
+      "$ref": "#/definitions/typeName"
     },
     "fullyQualifiedName": {
       "description": "FullyQualifiedName same as `name`.",
       "$ref": "../type/basic.json#/definitions/fullyQualifiedEntityName"
     },
-    "nameSpace": {
-      "description": "Namespace or group to which this type belongs to.",
-      "type": "string",
-      "default" : "custom"
-    },
     "displayName": {
       "description": "Display Name that identifies this type.",
       "type": "string"
     },
     "description": {
       "description": "Optional description of entity.",
-      "type": "string"
+      "$ref" : "../type/basic.json#/definitions/markdown"
+    },
+    "nameSpace": {
+      "description": "Namespace or group to which this type belongs to. For example, some of the field types commonly used can come from `basic` namespace. Some of the entities such as `table`, `database`, etc. come from `data` namespace.",
+      "type": "string",
+      "default" : "custom"
+    },
+    "category" : {
+      "$ref" : "#/definitions/category"
     },
     "schema": {
       "description": "JSON schema encoded as string that defines the type. This will be used to validate the type values.",
       "$ref": "../type/basic.json#/definitions/jsonSchema"
     },
+    "customFields" : {
+      "description": "Custom fields added to extend the entity. Only available for entity type",
+      "type" : "array",
+      "items" : {
+        "$ref" : "#/definitions/customField"
+      }
+    },
     "version": {
       "description": "Metadata version of the entity.",
       "$ref": "../type/entityHistory.json#/definitions/entityVersion"
@@ -61,7 +124,7 @@
     "name",
     "nameSpace",
     "description",
-    "type"
+    "schema"
   ],
   "additionalProperties": false
 }

catalog-rest-service/src/main/resources/json/schema/type/basic.json

@@ -5,17 +5,17 @@
   "description": "This schema defines basic common types that are used by other schemas.",
   "definitions": {
     "integer" : {
-      "$comment" : "@om-type",
+      "$comment" : "@om-field-type",
       "description": "An integer type.",
       "type" : "integer"
     },
     "number" : {
-      "$comment" : "@om-type",
+      "$comment" : "@om-field-type",
       "description": "A numeric type that includes integer or floating point numbers.",
       "type" : "integer"
     },
     "string" : {
-      "$comment" : "@om-type",
+      "$comment" : "@om-field-type",
       "description": "A String type.",
       "type" : "string"
     },
@@ -25,7 +25,7 @@
       "format": "uuid"
     },
     "email": {
-      "$comment" : "@om-type",
+      "$comment" : "@om-field-type",
       "description": "Email address of a user or other entities.",
       "type": "string",
       "format": "email",
@@ -34,7 +34,7 @@
       "maxLength": 127
     },
     "timestamp": {
-      "$comment" : "@om-type",
+      "$comment" : "@om-field-type",
       "description": "Timestamp in Unix epoch time milliseconds.",
       "@comment": "Note that during code generation this is converted into long",
       "type": "integer",
@@ -46,7 +46,7 @@
       "format": "uri"
     },
     "timeInterval": {
-      "$comment" : "@om-type",
+      "$comment" : "@om-field-type",
       "type": "object",
       "description": "Time interval in unixTimeMillis.",
       "javaType": "org.openmetadata.catalog.type.TimeInterval",
@@ -63,18 +63,18 @@
       "additionalProperties": false
     },
     "duration": {
-      "$comment" : "@om-type",
+      "$comment" : "@om-field-type",
       "description": "Duration in ISO 8601 format in UTC. Example - 'P23DT23H'.",
       "type": "string"
     },
     "date": {
-      "$comment" : "@om-type",
+      "$comment" : "@om-field-type",
       "description": "Date in ISO 8601 format in UTC. Example - '2018-11-13'.",
       "type": "string",
       "format": "date"
     },
     "dateTime": {
-      "$comment" : "@om-type",
+      "$comment" : "@om-field-type",
       "description": "Date and time in ISO 8601 format. Example - '2018-11-13T20:20:39+00:00'.",
       "type": "string",
       "format": "date-time"
@@ -97,12 +97,12 @@
       "maxLength": 256
     },
     "sqlQuery": {
-      "$comment" : "@om-type",
+      "$comment" : "@om-field-type",
       "description": "SQL query statement. Example - 'select * from orders'.",
       "type": "string"
     },
     "markdown": {
-      "$comment" : "@om-type",
+      "$comment" : "@om-field-type",
       "description": "Text in Markdown format",
       "type": "string"
     },

catalog-rest-service/src/test/java/org/openmetadata/catalog/CatalogApplicationTest.java

@@ -49,6 +49,7 @@ public abstract class CatalogApplicationTest {
     // The system properties are provided by maven-surefire for testing with mysql and postgres
     final String jdbcContainerClassName = System.getProperty("jdbcContainerClassName");
     final String jdbcContainerImage = System.getProperty("jdbcContainerImage");
+    LOG.info("Using test container class {} and image {}", jdbcContainerClassName, jdbcContainerImage);
 
     SQL_CONTAINER =
         (JdbcDatabaseContainer<?>)

catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java

@@ -145,6 +145,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K> extends C
   protected boolean supportsAuthorizedMetadataOperations = true;
   protected boolean supportsFieldsQueryParam = true;
   protected boolean supportsEmptyDescription = true;
+  protected boolean supportsNameWithDot = true;
 
   public static final String DATA_STEWARD_ROLE_NAME = "DataSteward";
   public static final String DATA_CONSUMER_ROLE_NAME = "DataConsumer";
@@ -623,14 +624,14 @@ public abstract class EntityResourceTest<T extends EntityInterface, K> extends C
   // Common entity tests for POST operations
   ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
   @Test
-  void post_entityCreateWithInvalidName_400() {
+  protected void post_entityCreateWithInvalidName_400() {
     // Create an entity with mandatory name field null
     final K request = createRequest(null, "description", "displayName", null);
     assertResponse(() -> createEntity(request, ADMIN_AUTH_HEADERS), BAD_REQUEST, "[name must not be null]");
 
     // Create an entity with mandatory name field empty
     final K request1 = createRequest("", "description", "displayName", null);
-    assertResponse(() -> createEntity(request1, ADMIN_AUTH_HEADERS), BAD_REQUEST, ENTITY_NAME_LENGTH_ERROR);
+    assertResponseContains(() -> createEntity(request1, ADMIN_AUTH_HEADERS), BAD_REQUEST, ENTITY_NAME_LENGTH_ERROR);
 
     // Create an entity with mandatory name field too long
     final K request2 = createRequest(LONG_ENTITY_NAME, "description", "displayName", null);
@@ -698,6 +699,9 @@ public abstract class EntityResourceTest<T extends EntityInterface, K> extends C
 
   @Test
   void post_entityWithDots_200() throws HttpResponseException {
+    if (!supportsNameWithDot) {
+      return;
+    }
     // Entity without "." should not have quoted fullyQualifiedName
     String name = format("%s_foo_bar", entityType);
     K request = createRequest(name, "", null, null);

catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/metadata/TypeResourceTest.java

@@ -14,20 +14,30 @@
 package org.openmetadata.catalog.resources.metadata;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
+import static org.openmetadata.catalog.util.TestUtils.assertResponse;
+import static org.openmetadata.catalog.util.TestUtils.assertResponseContains;
+import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.util.Map;
+import java.util.UUID;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Response.Status;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.client.HttpResponseException;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.TestMethodOrder;
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.api.CreateType;
 import org.openmetadata.catalog.entity.Type;
+import org.openmetadata.catalog.entity.type.Category;
+import org.openmetadata.catalog.entity.type.CustomField;
 import org.openmetadata.catalog.resources.EntityResourceTest;
 import org.openmetadata.catalog.resources.types.TypeResource;
 import org.openmetadata.catalog.resources.types.TypeResource.TypeList;
@@ -43,12 +53,63 @@ public class TypeResourceTest extends EntityResourceTest<Type, CreateType> {
     super(Entity.TYPE, Type.class, TypeList.class, "metadata/types", TypeResource.FIELDS);
     supportsEmptyDescription = false;
     supportsFieldsQueryParam = false;
+    supportsNameWithDot = false;
   }
 
   @BeforeAll
   public void setup(TestInfo test) throws IOException, URISyntaxException {
     super.setup(test);
-    INT_TYPE = getEntityByName("type.basic.integer", "", ADMIN_AUTH_HEADERS);
+    INT_TYPE = getEntityByName("integer", "", ADMIN_AUTH_HEADERS);
+  }
+
+  @Override
+  @Test
+  public void post_entityCreateWithInvalidName_400() {
+    String[][] tests = {
+      {"Abcd", "[name must match \"^[a-z][a-zA-Z0-9]+$\"]"},
+      {"a bc", "[name must match \"^[a-z][a-zA-Z0-9]+$\"]"}, // Name must not have space
+      {"a_bc", "[name must match \"^[a-z][a-zA-Z0-9]+$\"]"}, // Name must not be underscored
+      {"a-bc", "[name must match \"^[a-z][a-zA-Z0-9]+$\"]"}, // Name must not be hyphened
+    };
+
+    CreateType create = createRequest("placeHolder", "", "", null);
+    for (String[] test : tests) {
+      LOG.info("Testing with the name {}", test[0]);
+      create.withName(test[0]);
+      assertResponseContains(() -> createEntity(create, ADMIN_AUTH_HEADERS), Status.BAD_REQUEST, test[1]);
+    }
+  }
+
+  @Test
+  public void put_customField_200() throws HttpResponseException {
+    Type tableEntity = getEntityByName("table", "customFields", ADMIN_AUTH_HEADERS);
+    assertTrue(listOrEmpty(tableEntity.getCustomFields()).isEmpty());
+
+    // Add a custom field with name intA with type integer
+    CustomField fieldA =
+        new CustomField().withName("intA").withDescription("intA").withFieldType(INT_TYPE.getEntityReference());
+    tableEntity = addCustomField(tableEntity.getId(), fieldA, Status.OK, ADMIN_AUTH_HEADERS);
+    assertEquals(1, tableEntity.getCustomFields().size());
+    assertEquals(fieldA, tableEntity.getCustomFields().get(0));
+
+    // Add a second field with name intB with type integer
+    CustomField fieldB =
+        new CustomField().withName("intB").withDescription("intB").withFieldType(INT_TYPE.getEntityReference());
+    tableEntity = addCustomField(tableEntity.getId(), fieldB, Status.OK, ADMIN_AUTH_HEADERS);
+    assertEquals(2, tableEntity.getCustomFields().size());
+    assertEquals(fieldA, tableEntity.getCustomFields().get(0));
+    assertEquals(fieldB, tableEntity.getCustomFields().get(1));
+  }
+
+  @Test
+  public void put_customFieldToFieldType_4xx() {
+    // Adding a custom field to a field type is not allowed (only entity type is allowed)
+    CustomField field =
+        new CustomField().withName("intA").withDescription("intA").withFieldType(INT_TYPE.getEntityReference());
+    assertResponse(
+        () -> addCustomField(INT_TYPE.getId(), field, Status.CREATED, ADMIN_AUTH_HEADERS),
+        Status.BAD_REQUEST,
+        "Field types can't be extended");
   }
 
   @Override
@@ -61,12 +122,23 @@ public class TypeResourceTest extends EntityResourceTest<Type, CreateType> {
     return type;
   }
 
+  public Type addCustomField(UUID entityTypeId, CustomField customField, Status status, Map<String, String> authHeaders)
+      throws HttpResponseException {
+    WebTarget target = getResource(entityTypeId);
+    return TestUtils.put(target, customField, Type.class, status, authHeaders);
+  }
+
   @Override
   public CreateType createRequest(String name, String description, String displayName, EntityReference owner) {
+    if (name != null) {
+      name = name.replaceAll("[. _-]", "");
+    }
+    System.out.println("XXX Using the name " + name);
     return new CreateType()
         .withName(name)
         .withDescription(description)
         .withDisplayName(displayName)
+        .withCategory(Category.Field)
         .withSchema(INT_TYPE.getSchema());
   }