mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-11-02 11:39:12 +00:00
This commit is contained in:
Suresh Srinivas
GitHub
parent
06b062c1f4
commit
0ba22c1d2f
@ -1,18 +1,12 @@
|
||||
--
|
||||
-- Table to be used for generic entities that are limited in number. Examples of generic entities are
|
||||
-- Attribute entity, domain entities etc.
|
||||
--
|
||||
-- This reduces need for defining a table per entity.
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS generic_entity (
|
||||
CREATE TABLE IF NOT EXISTS type_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
-- Fully qualified name formed by entityType + "." + entityName
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
category VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.category') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
ALTER TABLE webhook_entity
|
||||
|
||||
@ -1,18 +1,12 @@
|
||||
--
|
||||
-- Table to be used for generic entities that are limited in number. Examples of generic entities are
|
||||
-- Attribute entity, domain entities etc.
|
||||
--
|
||||
-- This reduces need for defining a table per entity.
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS generic_entity (
|
||||
CREATE TABLE IF NOT EXISTS type_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
-- Fully qualified name formed by entityType + "." + entityName
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
category VARCHAR(256) GENERATED ALWAYS AS (json ->> 'category') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
ALTER TABLE webhook_entity
|
||||
|
||||
@ -486,6 +486,9 @@
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>${maven-surefire.version}</version>
|
||||
<configuration>
|
||||
<trimStackTrace>false</trimStackTrace>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>default-test</id>
|
||||
|
||||
@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory;
|
||||
public class CatalogGenericExceptionMapper implements ExceptionMapper<Throwable> {
|
||||
@Override
|
||||
public Response toResponse(Throwable ex) {
|
||||
ex.printStackTrace();
|
||||
LOG.debug(ex.getMessage());
|
||||
if (ex instanceof ProcessingException || ex instanceof IllegalArgumentException) {
|
||||
final Response response = BadRequestException.of().getResponse();
|
||||
|
||||
@ -176,7 +176,7 @@ public interface CollectionDAO {
|
||||
WebhookDAO webhookDAO();
|
||||
|
||||
@CreateSqlObject
|
||||
GenericEntityDAO genericEntityDAO();
|
||||
TypeEntityDAO typeEntityDAO();
|
||||
|
||||
interface DashboardDAO extends EntityDAO<Dashboard> {
|
||||
@Override
|
||||
@ -339,16 +339,23 @@ public interface CollectionDAO {
|
||||
|
||||
interface EntityRelationshipDAO {
|
||||
default int insert(UUID fromId, UUID toId, String fromEntity, String toEntity, int relation) {
|
||||
return insert(fromId.toString(), toId.toString(), fromEntity, toEntity, relation);
|
||||
return insert(fromId, toId, fromEntity, toEntity, relation, null);
|
||||
}
|
||||
|
||||
default int insert(UUID fromId, UUID toId, String fromEntity, String toEntity, int relation, String json) {
|
||||
return insert(fromId.toString(), toId.toString(), fromEntity, toEntity, relation, json);
|
||||
}
|
||||
|
||||
@ConnectionAwareSqlUpdate(
|
||||
value =
|
||||
"INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation)",
|
||||
"INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) "
|
||||
+ "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation)",
|
||||
connectionType = MYSQL)
|
||||
@ConnectionAwareSqlUpdate(
|
||||
value =
|
||||
"INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation) ON CONFLICT (fromId, toId, relation) DO NOTHING",
|
||||
"INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) "
|
||||
+ "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation) "
|
||||
+ "ON CONFLICT (fromId, toId, relation) DO NOTHING",
|
||||
connectionType = POSTGRES)
|
||||
int insert(
|
||||
@Bind("fromId") String fromId,
|
||||
@ -357,6 +364,25 @@ public interface CollectionDAO {
|
||||
@Bind("toEntity") String toEntity,
|
||||
@Bind("relation") int relation);
|
||||
|
||||
@ConnectionAwareSqlUpdate(
|
||||
value =
|
||||
"INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation, json) "
|
||||
+ "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation, :json)",
|
||||
connectionType = MYSQL)
|
||||
@ConnectionAwareSqlUpdate(
|
||||
value =
|
||||
"INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation, json) VALUES "
|
||||
+ "(:fromId, :toId, :fromEntity, :toEntity, :relation, (:json :: jsonb)) "
|
||||
+ "ON CONFLICT (fromId, toId, relation) DO NOTHING",
|
||||
connectionType = POSTGRES)
|
||||
int insert(
|
||||
@Bind("fromId") String fromId,
|
||||
@Bind("toId") String toId,
|
||||
@Bind("fromEntity") String fromEntity,
|
||||
@Bind("toEntity") String toEntity,
|
||||
@Bind("relation") int relation,
|
||||
@Bind("json") String json);
|
||||
|
||||
//
|
||||
// Find to operations
|
||||
//
|
||||
@ -368,17 +394,6 @@ public interface CollectionDAO {
|
||||
List<EntityReference> findTo(
|
||||
@Bind("fromId") String fromId, @Bind("fromEntity") String fromEntity, @Bind("relation") int relation);
|
||||
|
||||
@SqlQuery(
|
||||
"SELECT toId, toEntity FROM entity_relationship "
|
||||
+ "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity "
|
||||
+ "ORDER BY toId")
|
||||
@RegisterRowMapper(ToEntityReferenceMapper.class)
|
||||
List<EntityReference> findToReference(
|
||||
@Bind("fromId") String fromId,
|
||||
@Bind("fromEntity") String fromEntity,
|
||||
@Bind("relation") int relation,
|
||||
@Bind("toEntity") String toEntity);
|
||||
|
||||
@SqlQuery(
|
||||
"SELECT toId FROM entity_relationship "
|
||||
+ "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity "
|
||||
@ -421,17 +436,6 @@ public interface CollectionDAO {
|
||||
List<EntityReference> findFrom(
|
||||
@Bind("toId") String toId, @Bind("toEntity") String toEntity, @Bind("relation") int relation);
|
||||
|
||||
@SqlQuery(
|
||||
"SELECT fromId, fromEntity FROM entity_relationship "
|
||||
+ "WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation AND fromEntity = :fromEntity "
|
||||
+ "ORDER BY fromId")
|
||||
@RegisterRowMapper(FromEntityReferenceMapper.class)
|
||||
List<EntityReference> findFromEntity(
|
||||
@Bind("toId") String toId,
|
||||
@Bind("toEntity") String toEntity,
|
||||
@Bind("relation") int relation,
|
||||
@Bind("fromEntity") String fromEntity);
|
||||
|
||||
//
|
||||
// Delete Operations
|
||||
//
|
||||
@ -709,18 +713,22 @@ public interface CollectionDAO {
|
||||
interface FieldRelationshipDAO {
|
||||
@ConnectionAwareSqlUpdate(
|
||||
value =
|
||||
"INSERT IGNORE INTO field_relationship(fromFQN, toFQN, fromType, toType, relation) VALUES (:fromFQN, :toFQN, :fromType, :toType, :relation)",
|
||||
"INSERT IGNORE INTO field_relationship(fromFQN, toFQN, fromType, toType, relation, json) "
|
||||
+ "VALUES (:fromFQN, :toFQN, :fromType, :toType, :relation, :json)",
|
||||
connectionType = MYSQL)
|
||||
@ConnectionAwareSqlUpdate(
|
||||
value =
|
||||
"INSERT INTO field_relationship(fromFQN, toFQN, fromType, toType, relation) VALUES (:fromFQN, :toFQN, :fromType, :toType, :relation) ON CONFLICT (fromFQN, toFQN, relation) DO NOTHING",
|
||||
"INSERT INTO field_relationship(fromFQN, toFQN, fromType, toType, relation, json) "
|
||||
+ "VALUES (:fromFQN, :toFQN, :fromType, :toType, :relation, (:json :: jsonb)) "
|
||||
+ "ON CONFLICT (fromFQN, toFQN, relation) DO NOTHING",
|
||||
connectionType = POSTGRES)
|
||||
void insert(
|
||||
int insert(
|
||||
@Bind("fromFQN") String fromFQN,
|
||||
@Bind("toFQN") String toFQN,
|
||||
@Bind("fromType") String fromType,
|
||||
@Bind("toType") String toType,
|
||||
@Bind("relation") int relation);
|
||||
@Bind("relation") int relation,
|
||||
@Bind("json") String json);
|
||||
|
||||
@ConnectionAwareSqlUpdate(
|
||||
value =
|
||||
@ -794,6 +802,16 @@ public interface CollectionDAO {
|
||||
@SqlUpdate("DELETE from field_relationship <cond>")
|
||||
void deleteAllByPrefixInternal(@Define("cond") String cond);
|
||||
|
||||
@SqlUpdate(
|
||||
"DELETE from field_relationship WHERE fromFQN = :fromFQN AND toFQN = :toFQN AND fromType = :fromType "
|
||||
+ "AND toType = :toType AND relation = :relation")
|
||||
void delete(
|
||||
@Bind("fromFQN") String fromFQN,
|
||||
@Bind("toFQN") String toFQN,
|
||||
@Bind("fromType") String fromType,
|
||||
@Bind("toType") String toType,
|
||||
@Bind("relation") int relation);
|
||||
|
||||
class ToFieldMapper implements RowMapper<List<String>> {
|
||||
@Override
|
||||
public List<String> map(ResultSet rs, StatementContext ctx) throws SQLException {
|
||||
@ -1560,10 +1578,10 @@ public interface CollectionDAO {
|
||||
List<String> listWithoutEntityFilter(@Bind("eventType") String eventType, @Bind("timestamp") long timestamp);
|
||||
}
|
||||
|
||||
interface GenericEntityDAO extends EntityDAO<Type> {
|
||||
interface TypeEntityDAO extends EntityDAO<Type> {
|
||||
@Override
|
||||
default String getTableName() {
|
||||
return "generic_entity";
|
||||
return "type_entity";
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -1573,7 +1591,7 @@ public interface CollectionDAO {
|
||||
|
||||
@Override
|
||||
default String getNameColumn() {
|
||||
return "fullyQualifiedName";
|
||||
return "name";
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -115,7 +115,7 @@ import org.openmetadata.catalog.util.ResultList;
|
||||
public abstract class EntityRepository<T extends EntityInterface> {
|
||||
private final String collectionPath;
|
||||
private final Class<T> entityClass;
|
||||
private final String entityType;
|
||||
protected final String entityType;
|
||||
public final EntityDAO<T> dao;
|
||||
protected final CollectionDAO daoCollection;
|
||||
protected final List<String> allowedFields;
|
||||
@ -129,7 +129,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
private final Fields patchFields;
|
||||
|
||||
/** Fields that can be updated during PUT operation */
|
||||
private final Fields putFields;
|
||||
protected final Fields putFields;
|
||||
|
||||
EntityRepository(
|
||||
String collectionPath,
|
||||
@ -737,8 +737,24 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
return addRelationship(fromId, toId, fromEntity, toEntity, relationship, false);
|
||||
}
|
||||
|
||||
public int addRelationship(
|
||||
UUID fromId, UUID toId, String fromEntity, String toEntity, Relationship relationship, String json) {
|
||||
return addRelationship(fromId, toId, fromEntity, toEntity, relationship, json, false);
|
||||
}
|
||||
|
||||
public int addRelationship(
|
||||
UUID fromId, UUID toId, String fromEntity, String toEntity, Relationship relationship, boolean bidirectional) {
|
||||
return addRelationship(fromId, toId, fromEntity, toEntity, relationship, null, bidirectional);
|
||||
}
|
||||
|
||||
public int addRelationship(
|
||||
UUID fromId,
|
||||
UUID toId,
|
||||
String fromEntity,
|
||||
String toEntity,
|
||||
Relationship relationship,
|
||||
String json,
|
||||
boolean bidirectional) {
|
||||
UUID from = fromId;
|
||||
UUID to = toId;
|
||||
if (bidirectional && fromId.compareTo(toId) > 0) {
|
||||
@ -747,7 +763,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
from = toId;
|
||||
to = fromId;
|
||||
}
|
||||
return daoCollection.relationshipDAO().insert(from, to, fromEntity, toEntity, relationship.ordinal());
|
||||
return daoCollection.relationshipDAO().insert(fromId, toId, fromEntity, toEntity, relationship.ordinal(), json);
|
||||
}
|
||||
|
||||
public List<String> findBoth(UUID entity1, String entityType1, Relationship relationship, String entity2) {
|
||||
|
||||
@ -13,6 +13,11 @@
|
||||
|
||||
package org.openmetadata.catalog.jdbi3;
|
||||
|
||||
import static org.openmetadata.catalog.type.Relationship.ADDRESSED_TO;
|
||||
import static org.openmetadata.catalog.type.Relationship.CREATED;
|
||||
import static org.openmetadata.catalog.type.Relationship.IS_ABOUT;
|
||||
import static org.openmetadata.catalog.type.Relationship.REPLIED_TO;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
@ -83,13 +88,7 @@ public class FeedRepository {
|
||||
dao.feedDAO().insert(JsonUtils.pojoToJson(thread));
|
||||
|
||||
// Add relationship User -- created --> Thread relationship
|
||||
dao.relationshipDAO()
|
||||
.insert(
|
||||
createdByUser.getId().toString(),
|
||||
thread.getId().toString(),
|
||||
Entity.USER,
|
||||
Entity.THREAD,
|
||||
Relationship.CREATED.ordinal());
|
||||
dao.relationshipDAO().insert(createdByUser.getId(), thread.getId(), Entity.USER, Entity.THREAD, CREATED.ordinal());
|
||||
|
||||
// Add field relationship data asset Thread -- isAbout ---> entity/entityField
|
||||
// relationship
|
||||
@ -99,17 +98,13 @@ public class FeedRepository {
|
||||
about.getFullyQualifiedFieldValue(), // to FQN
|
||||
Entity.THREAD, // From type
|
||||
about.getFullyQualifiedFieldType(), // to Type
|
||||
Relationship.IS_ABOUT.ordinal());
|
||||
IS_ABOUT.ordinal(),
|
||||
null);
|
||||
|
||||
// Add the owner also as addressedTo as the entity he owns when addressed, the owner is actually being addressed
|
||||
if (entityOwner != null) {
|
||||
dao.relationshipDAO()
|
||||
.insert(
|
||||
thread.getId().toString(),
|
||||
entityOwner.getId().toString(),
|
||||
Entity.THREAD,
|
||||
entityOwner.getType(),
|
||||
Relationship.ADDRESSED_TO.ordinal());
|
||||
.insert(thread.getId(), entityOwner.getId(), Entity.THREAD, entityOwner.getType(), ADDRESSED_TO.ordinal());
|
||||
}
|
||||
|
||||
// Add mentions to field relationship table
|
||||
@ -151,7 +146,8 @@ public class FeedRepository {
|
||||
thread.getId().toString(),
|
||||
mention.getFullyQualifiedFieldType(),
|
||||
Entity.THREAD,
|
||||
Relationship.MENTIONED_IN.ordinal()));
|
||||
Relationship.MENTIONED_IN.ordinal(),
|
||||
null));
|
||||
}
|
||||
|
||||
@Transaction
|
||||
@ -178,13 +174,7 @@ public class FeedRepository {
|
||||
}
|
||||
}
|
||||
if (!relationAlreadyExists) {
|
||||
dao.relationshipDAO()
|
||||
.insert(
|
||||
fromUser.getId().toString(),
|
||||
thread.getId().toString(),
|
||||
Entity.USER,
|
||||
Entity.THREAD,
|
||||
Relationship.REPLIED_TO.ordinal());
|
||||
dao.relationshipDAO().insert(fromUser.getId(), thread.getId(), Entity.USER, Entity.THREAD, REPLIED_TO.ordinal());
|
||||
}
|
||||
|
||||
// Add mentions into field relationship table
|
||||
@ -233,7 +223,7 @@ public class FeedRepository {
|
||||
result =
|
||||
dao.feedDAO()
|
||||
.listCountByEntityLink(
|
||||
StringUtils.EMPTY, Entity.THREAD, StringUtils.EMPTY, Relationship.IS_ABOUT.ordinal(), isResolved);
|
||||
StringUtils.EMPTY, Entity.THREAD, StringUtils.EMPTY, IS_ABOUT.ordinal(), isResolved);
|
||||
} else {
|
||||
EntityLink entityLink = EntityLink.parse(link);
|
||||
EntityReference reference = EntityUtil.validateEntityLink(entityLink);
|
||||
@ -253,7 +243,7 @@ public class FeedRepository {
|
||||
entityLink.getFullyQualifiedFieldValue(),
|
||||
Entity.THREAD,
|
||||
entityLink.getFullyQualifiedFieldType(),
|
||||
Relationship.IS_ABOUT.ordinal(),
|
||||
IS_ABOUT.ordinal(),
|
||||
isResolved);
|
||||
}
|
||||
}
|
||||
@ -333,7 +323,7 @@ public class FeedRepository {
|
||||
limit + 1,
|
||||
time,
|
||||
isResolved,
|
||||
Relationship.IS_ABOUT.ordinal());
|
||||
IS_ABOUT.ordinal());
|
||||
} else {
|
||||
jsons =
|
||||
dao.feedDAO()
|
||||
@ -343,7 +333,7 @@ public class FeedRepository {
|
||||
limit + 1,
|
||||
time,
|
||||
isResolved,
|
||||
Relationship.IS_ABOUT.ordinal());
|
||||
IS_ABOUT.ordinal());
|
||||
}
|
||||
threads = JsonUtils.readObjects(jsons, Thread.class);
|
||||
total =
|
||||
@ -352,7 +342,7 @@ public class FeedRepository {
|
||||
entityLink.getFullyQualifiedFieldValue(),
|
||||
entityLink.getFullyQualifiedFieldType(),
|
||||
isResolved,
|
||||
Relationship.IS_ABOUT.ordinal());
|
||||
IS_ABOUT.ordinal());
|
||||
}
|
||||
} else {
|
||||
FilteredThreads filteredThreads;
|
||||
@ -474,7 +464,7 @@ public class FeedRepository {
|
||||
String userId, int limit, long time, boolean isResolved, PaginationType paginationType) throws IOException {
|
||||
List<EntityReference> teams =
|
||||
EntityUtil.populateEntityReferences(
|
||||
dao.relationshipDAO().findFromEntity(userId, Entity.USER, Relationship.HAS.ordinal(), Entity.TEAM));
|
||||
dao.relationshipDAO().findFrom(userId, Entity.USER, Relationship.HAS.ordinal(), Entity.TEAM), Entity.TEAM);
|
||||
List<String> teamNames = teams.stream().map(EntityReference::getName).collect(Collectors.toList());
|
||||
if (teamNames.isEmpty()) {
|
||||
teamNames = List.of(StringUtils.EMPTY);
|
||||
|
||||
@ -25,9 +25,6 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.jdbi.v3.sqlobject.transaction.Transaction;
|
||||
import org.openmetadata.catalog.Entity;
|
||||
import org.openmetadata.catalog.entity.teams.Role;
|
||||
import org.openmetadata.catalog.entity.teams.User;
|
||||
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
|
||||
import org.openmetadata.catalog.exception.EntityNotFoundException;
|
||||
import org.openmetadata.catalog.resources.teams.RoleResource;
|
||||
import org.openmetadata.catalog.type.EntityReference;
|
||||
import org.openmetadata.catalog.type.Relationship;
|
||||
@ -178,18 +175,5 @@ public class RoleRepository extends EntityRepository<Role> {
|
||||
.relationshipDAO()
|
||||
.deleteTo(role.getId().toString(), Entity.ROLE, Relationship.HAS.ordinal(), Entity.USER);
|
||||
}
|
||||
|
||||
private List<User> getAllUsers() {
|
||||
EntityRepository<User> userRepository = Entity.getEntityRepository(Entity.USER);
|
||||
try {
|
||||
// Assumptions:
|
||||
// - we will not have more than Integer.MAX_VALUE users in the system.
|
||||
// - we do not need to update deleted user's roles.
|
||||
ListFilter filter = new ListFilter();
|
||||
return userRepository.listAfter(null, Fields.EMPTY_FIELDS, filter, Integer.MAX_VALUE - 1, null).getData();
|
||||
} catch (IOException e) {
|
||||
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entitiesNotFound(Entity.USER));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,34 +16,43 @@
|
||||
|
||||
package org.openmetadata.catalog.jdbi3;
|
||||
|
||||
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import javax.ws.rs.core.UriInfo;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.openmetadata.catalog.Entity;
|
||||
import org.openmetadata.catalog.entity.Type;
|
||||
import org.openmetadata.catalog.entity.type.Category;
|
||||
import org.openmetadata.catalog.entity.type.CustomField;
|
||||
import org.openmetadata.catalog.resources.types.TypeResource;
|
||||
import org.openmetadata.catalog.type.Include;
|
||||
import org.openmetadata.catalog.type.Relationship;
|
||||
import org.openmetadata.catalog.util.EntityUtil;
|
||||
import org.openmetadata.catalog.util.EntityUtil.Fields;
|
||||
import org.openmetadata.catalog.util.FullyQualifiedName;
|
||||
import org.openmetadata.catalog.util.JsonUtils;
|
||||
import org.openmetadata.catalog.util.RestUtil.PutResponse;
|
||||
|
||||
@Slf4j
|
||||
public class TypeRepository extends EntityRepository<Type> {
|
||||
// TODO fix this
|
||||
private static final String UPDATE_FIELDS = "";
|
||||
private static final String PATCH_FIELDS = "";
|
||||
private static final String UPDATE_FIELDS = "customFields";
|
||||
private static final String PATCH_FIELDS = "customFields";
|
||||
|
||||
public TypeRepository(CollectionDAO dao) {
|
||||
super(
|
||||
TypeResource.COLLECTION_PATH,
|
||||
Entity.TYPE,
|
||||
Type.class,
|
||||
dao.genericEntityDAO(),
|
||||
dao,
|
||||
PATCH_FIELDS,
|
||||
UPDATE_FIELDS);
|
||||
super(TypeResource.COLLECTION_PATH, Entity.TYPE, Type.class, dao.typeEntityDAO(), dao, PATCH_FIELDS, UPDATE_FIELDS);
|
||||
allowEdits = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Type setFields(Type attribute, Fields fields) throws IOException {
|
||||
return attribute;
|
||||
public Type setFields(Type type, Fields fields) throws IOException {
|
||||
type.withCustomFields(fields.contains("customFields") ? getCustomFields(type) : null);
|
||||
return type;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -54,9 +63,10 @@ public class TypeRepository extends EntityRepository<Type> {
|
||||
@Override
|
||||
public void storeEntity(Type type, boolean update) throws IOException {
|
||||
URI href = type.getHref();
|
||||
type.withHref(null);
|
||||
List<CustomField> customFields = type.getCustomFields();
|
||||
type.withHref(null).withCustomFields(null);
|
||||
store(type.getId(), type, update);
|
||||
type.withHref(href);
|
||||
type.withHref(href).withCustomFields(customFields);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -65,19 +75,95 @@ public class TypeRepository extends EntityRepository<Type> {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setFullyQualifiedName(Type entity) {
|
||||
entity.setFullyQualifiedName(FullyQualifiedName.build(Entity.TYPE, entity.getNameSpace(), entity.getName()));
|
||||
public EntityUpdater getUpdater(Type original, Type updated, Operation operation) {
|
||||
return new TypeUpdater(original, updated, operation);
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntityUpdater getUpdater(Type original, Type updated, Operation operation) {
|
||||
return new AttributeUpdater(original, updated, operation);
|
||||
public PutResponse<Type> addCustomField(UriInfo uriInfo, String updatedBy, String id, CustomField field)
|
||||
throws IOException {
|
||||
Type type = dao.findEntityById(UUID.fromString(id), Include.NON_DELETED);
|
||||
if (type.getCategory().equals(Category.Field)) {
|
||||
throw new IllegalArgumentException("Field types can't be extended");
|
||||
}
|
||||
setFields(type, putFields);
|
||||
|
||||
dao.findEntityById(field.getFieldType().getId()); // Validate customField type exists
|
||||
type.getCustomFields().add(field);
|
||||
type.setUpdatedBy(updatedBy);
|
||||
type.setUpdatedAt(System.currentTimeMillis());
|
||||
return createOrUpdate(uriInfo, type);
|
||||
}
|
||||
|
||||
private String getCustomFieldFQNPrefix(Type type) {
|
||||
return FullyQualifiedName.build(type.getName(), "customFields");
|
||||
}
|
||||
|
||||
private String getCustomFieldFQN(Type type, String fieldName) {
|
||||
return FullyQualifiedName.build(type.getName(), "customFields", fieldName);
|
||||
}
|
||||
|
||||
private List<CustomField> getCustomFields(Type type) throws IOException {
|
||||
List<CustomField> customFields = new ArrayList<>();
|
||||
List<List<String>> results =
|
||||
daoCollection
|
||||
.fieldRelationshipDAO()
|
||||
.listToByPrefix(getCustomFieldFQNPrefix(type), Entity.TYPE, Entity.TYPE, Relationship.HAS.ordinal());
|
||||
for (List<String> result : results) {
|
||||
CustomField field = JsonUtils.readValue(result.get(2), CustomField.class);
|
||||
field.setFieldType(dao.findEntityReferenceByName(result.get(1)));
|
||||
customFields.add(field);
|
||||
}
|
||||
customFields.sort(EntityUtil.compareCustomField);
|
||||
return customFields;
|
||||
}
|
||||
|
||||
/** Handles entity updated from PUT and POST operation. */
|
||||
public class AttributeUpdater extends EntityUpdater {
|
||||
public AttributeUpdater(Type original, Type updated, Operation operation) {
|
||||
public class TypeUpdater extends EntityUpdater {
|
||||
public TypeUpdater(Type original, Type updated, Operation operation) {
|
||||
super(original, updated, operation);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void entitySpecificUpdate() throws IOException {
|
||||
updateCustomFields();
|
||||
}
|
||||
|
||||
private void updateCustomFields() throws JsonProcessingException {
|
||||
List<CustomField> updatedFields = listOrEmpty(updated.getCustomFields());
|
||||
List<CustomField> origFields = listOrEmpty(original.getCustomFields());
|
||||
List<CustomField> added = new ArrayList<>();
|
||||
List<CustomField> deleted = new ArrayList<>();
|
||||
recordListChange("charts", origFields, updatedFields, added, deleted, EntityUtil.customFieldMatch);
|
||||
for (CustomField field : added) {
|
||||
String customFieldFQN = getCustomFieldFQN(updated, field.getName());
|
||||
String customFieldJson = JsonUtils.pojoToJson(field);
|
||||
LOG.info(
|
||||
"Adding customField {} with type {} to the entity {}",
|
||||
field.getName(),
|
||||
field.getFieldType().getName(),
|
||||
updated.getName());
|
||||
daoCollection
|
||||
.fieldRelationshipDAO()
|
||||
.insert(
|
||||
customFieldFQN,
|
||||
field.getFieldType().getName(),
|
||||
Entity.TYPE,
|
||||
Entity.TYPE,
|
||||
Relationship.HAS.ordinal(),
|
||||
customFieldJson);
|
||||
}
|
||||
for (CustomField field : deleted) {
|
||||
String customFieldFQN = getCustomFieldFQN(updated, field.getName());
|
||||
LOG.info(
|
||||
"Deleting customField {} with type {} from the entity {}",
|
||||
field.getName(),
|
||||
field.getFieldType().getName(),
|
||||
updated.getName());
|
||||
daoCollection
|
||||
.fieldRelationshipDAO()
|
||||
.delete(
|
||||
customFieldFQN, field.getFieldType().getName(), Entity.TYPE, Entity.TYPE, Relationship.HAS.ordinal());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,6 +16,7 @@ package org.openmetadata.catalog.resources.types;
|
||||
import static org.openmetadata.catalog.security.SecurityUtil.ADMIN;
|
||||
import static org.openmetadata.catalog.security.SecurityUtil.BOT;
|
||||
import static org.openmetadata.catalog.security.SecurityUtil.OWNER;
|
||||
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
import io.swagger.annotations.Api;
|
||||
@ -52,18 +53,22 @@ import javax.ws.rs.core.SecurityContext;
|
||||
import javax.ws.rs.core.UriInfo;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.openmetadata.catalog.CatalogApplicationConfig;
|
||||
import org.openmetadata.catalog.Entity;
|
||||
import org.openmetadata.catalog.api.CreateType;
|
||||
import org.openmetadata.catalog.entity.Type;
|
||||
import org.openmetadata.catalog.entity.type.Category;
|
||||
import org.openmetadata.catalog.entity.type.CustomField;
|
||||
import org.openmetadata.catalog.jdbi3.CollectionDAO;
|
||||
import org.openmetadata.catalog.jdbi3.ListFilter;
|
||||
import org.openmetadata.catalog.jdbi3.TypeRepository;
|
||||
import org.openmetadata.catalog.resources.Collection;
|
||||
import org.openmetadata.catalog.resources.EntityResource;
|
||||
import org.openmetadata.catalog.security.Authorizer;
|
||||
import org.openmetadata.catalog.security.SecurityUtil;
|
||||
import org.openmetadata.catalog.type.EntityHistory;
|
||||
import org.openmetadata.catalog.type.Include;
|
||||
import org.openmetadata.catalog.util.EntityUtil;
|
||||
import org.openmetadata.catalog.util.JsonUtils;
|
||||
import org.openmetadata.catalog.util.RestUtil.PutResponse;
|
||||
import org.openmetadata.catalog.util.ResultList;
|
||||
|
||||
@Path("/v1/metadata/types")
|
||||
@ -77,7 +82,8 @@ public class TypeResource extends EntityResource<Type, TypeRepository> {
|
||||
|
||||
@Override
|
||||
public Type addHref(UriInfo uriInfo, Type type) {
|
||||
return type; // Nothing to do
|
||||
listOrEmpty(type.getCustomFields()).forEach(field -> Entity.withHref(uriInfo, field.getFieldType()));
|
||||
return type;
|
||||
}
|
||||
|
||||
@Inject
|
||||
@ -88,26 +94,18 @@ public class TypeResource extends EntityResource<Type, TypeRepository> {
|
||||
@SuppressWarnings("unused") // Method used for reflection
|
||||
public void initialize(CatalogApplicationConfig config) throws IOException {
|
||||
// Find tag definitions and load tag categories from the json file, if necessary
|
||||
List<String> jsonSchemas = EntityUtil.getJsonDataResources(".*json/schema/type/.*\\.json$");
|
||||
long now = System.currentTimeMillis();
|
||||
for (String jsonSchema : jsonSchemas) {
|
||||
try {
|
||||
List<Type> types = JsonUtils.getTypes(jsonSchema);
|
||||
types.forEach(
|
||||
type -> {
|
||||
type.withId(UUID.randomUUID()).withUpdatedBy("admin").withUpdatedAt(now);
|
||||
LOG.info("Loading from {} type {} with schema {}", jsonSchema, type.getName(), type.getSchema());
|
||||
try {
|
||||
this.dao.createOrUpdate(null, type);
|
||||
} catch (IOException e) {
|
||||
LOG.error("Error loading type {} from {}", type.getName(), jsonSchema, e);
|
||||
e.printStackTrace();
|
||||
}
|
||||
});
|
||||
} catch (Exception e) {
|
||||
LOG.warn("Failed to initialize the types from jsonSchema file {}", jsonSchema, e);
|
||||
}
|
||||
}
|
||||
List<Type> types = JsonUtils.getTypes();
|
||||
types.forEach(
|
||||
type -> {
|
||||
type.withId(UUID.randomUUID()).withUpdatedBy("admin").withUpdatedAt(now);
|
||||
LOG.info("Loading type {} with schema {}", type.getName(), type.getSchema());
|
||||
try {
|
||||
this.dao.createOrUpdate(null, type);
|
||||
} catch (IOException e) {
|
||||
LOG.error("Error loading type {}", type.getName(), e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public static class TypeList extends ResultList<Type> {
|
||||
@ -121,7 +119,7 @@ public class TypeResource extends EntityResource<Type, TypeRepository> {
|
||||
}
|
||||
}
|
||||
|
||||
public static final String FIELDS = "";
|
||||
public static final String FIELDS = "customFields";
|
||||
|
||||
@GET
|
||||
@Valid
|
||||
@ -141,6 +139,11 @@ public class TypeResource extends EntityResource<Type, TypeRepository> {
|
||||
public ResultList<Type> list(
|
||||
@Context UriInfo uriInfo,
|
||||
@Context SecurityContext securityContext,
|
||||
@Parameter(
|
||||
description = "Filter types by metadata type category.",
|
||||
schema = @Schema(type = "string", example = "Field, Entity"))
|
||||
@QueryParam("category")
|
||||
Category category,
|
||||
@Parameter(description = "Limit the number types returned. (1 to 1000000, " + "default = 10)")
|
||||
@DefaultValue("10")
|
||||
@Min(0)
|
||||
@ -349,11 +352,35 @@ public class TypeResource extends EntityResource<Type, TypeRepository> {
|
||||
return delete(uriInfo, securityContext, id, false, true, ADMIN | BOT);
|
||||
}
|
||||
|
||||
@PUT
|
||||
@Path("/{id}")
|
||||
@Operation(
|
||||
summary = "Add a field to an entity",
|
||||
tags = "metadata",
|
||||
description = "Add a field to an entity type. Fields can only be added to entity type and not field type.",
|
||||
responses = {
|
||||
@ApiResponse(responseCode = "200", description = "OK"),
|
||||
@ApiResponse(responseCode = "404", description = "type for instance {id} is not found")
|
||||
})
|
||||
public Response addField(
|
||||
@Context UriInfo uriInfo,
|
||||
@Context SecurityContext securityContext,
|
||||
@Parameter(description = "Type Id", schema = @Schema(type = "string")) @PathParam("id") String id,
|
||||
CustomField field)
|
||||
throws IOException {
|
||||
SecurityUtil.authorizeAdmin(authorizer, securityContext, ADMIN | BOT);
|
||||
PutResponse<Type> response = dao.addCustomField(uriInfo, securityContext.getUserPrincipal().getName(), id, field);
|
||||
addHref(uriInfo, response.getEntity());
|
||||
return response.toResponse();
|
||||
}
|
||||
|
||||
private Type getType(SecurityContext securityContext, CreateType create) {
|
||||
return new Type()
|
||||
.withId(UUID.randomUUID())
|
||||
.withName(create.getName())
|
||||
.withFullyQualifiedName(create.getName())
|
||||
.withDisplayName(create.getDisplayName())
|
||||
.withCategory(create.getCategory())
|
||||
.withSchema(create.getSchema())
|
||||
.withDescription(create.getDescription())
|
||||
.withUpdatedBy(securityContext.getUserPrincipal().getName())
|
||||
|
||||
@ -40,6 +40,7 @@ import org.openmetadata.catalog.EntityInterface;
|
||||
import org.openmetadata.catalog.api.data.TermReference;
|
||||
import org.openmetadata.catalog.entity.data.GlossaryTerm;
|
||||
import org.openmetadata.catalog.entity.data.Table;
|
||||
import org.openmetadata.catalog.entity.type.CustomField;
|
||||
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
|
||||
import org.openmetadata.catalog.exception.EntityNotFoundException;
|
||||
import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityVersionPair;
|
||||
@ -80,6 +81,7 @@ public final class EntityUtil {
|
||||
Comparator.comparing(TableConstraint::getConstraintType);
|
||||
public static final Comparator<ChangeEvent> compareChangeEvent = Comparator.comparing(ChangeEvent::getTimestamp);
|
||||
public static final Comparator<GlossaryTerm> compareGlossaryTerm = Comparator.comparing(GlossaryTerm::getName);
|
||||
public static final Comparator<CustomField> compareCustomField = Comparator.comparing(CustomField::getName);
|
||||
|
||||
//
|
||||
// Matchers used for matching two items in a list
|
||||
@ -130,6 +132,9 @@ public final class EntityUtil {
|
||||
public static final BiPredicate<TermReference, TermReference> termReferenceMatch =
|
||||
(ref1, ref2) -> ref1.getName().equals(ref2.getName()) && ref1.getEndpoint().equals(ref2.getEndpoint());
|
||||
|
||||
public static final BiPredicate<CustomField, CustomField> customFieldMatch =
|
||||
(ref1, ref2) -> ref1.getName().equals(ref2.getName());
|
||||
|
||||
private EntityUtil() {}
|
||||
|
||||
/** Validate Ingestion Schedule */
|
||||
|
||||
@ -45,10 +45,14 @@ import javax.json.JsonReader;
|
||||
import javax.json.JsonStructure;
|
||||
import javax.json.JsonValue;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.openmetadata.catalog.entity.Type;
|
||||
import org.openmetadata.catalog.entity.type.Category;
|
||||
|
||||
@Slf4j
|
||||
public final class JsonUtils {
|
||||
public static final String TYPE_ANNOTATION = "@om-type";
|
||||
public static final String FIELD_TYPE_ANNOTATION = "@om-field-type";
|
||||
public static final String ENTITY_TYPE_ANNOTATION = "@om-entity-type";
|
||||
public static final MediaType DEFAULT_MEDIA_TYPE = MediaType.APPLICATION_JSON_TYPE;
|
||||
private static final ObjectMapper OBJECT_MAPPER;
|
||||
private static final JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(VersionFlag.V7);
|
||||
@ -217,11 +221,39 @@ public final class JsonUtils {
|
||||
return comment != null && comment.contains(annotation);
|
||||
}
|
||||
|
||||
/** Get all the fields types and entity types from OpenMetadata JSON schema definition files. */
|
||||
public static List<Type> getTypes() throws IOException {
|
||||
// Get Field Types
|
||||
List<Type> types = new ArrayList<>();
|
||||
List<String> jsonSchemas = EntityUtil.getJsonDataResources(".*json/schema/type/.*\\.json$");
|
||||
for (String jsonSchema : jsonSchemas) {
|
||||
try {
|
||||
types.addAll(JsonUtils.getFieldTypes(jsonSchema));
|
||||
} catch (Exception e) {
|
||||
LOG.warn("Failed to initialize the types from jsonSchema file {}", jsonSchema, e);
|
||||
}
|
||||
}
|
||||
|
||||
// Get Entity Types
|
||||
jsonSchemas = EntityUtil.getJsonDataResources(".*json/schema/entity/.*\\.json$");
|
||||
for (String jsonSchema : jsonSchemas) {
|
||||
try {
|
||||
Type entityType = JsonUtils.getEntityType(jsonSchema);
|
||||
if (entityType != null) {
|
||||
types.add(entityType);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOG.warn("Failed to initialize the types from jsonSchema file {}", jsonSchema, e);
|
||||
}
|
||||
}
|
||||
return types;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all the types from the `definitions` section of a JSON schema file that are annotated with "$comment" field set
|
||||
* to "@om-type".
|
||||
* Get all the fields types from the `definitions` section of a JSON schema file that are annotated with "$comment"
|
||||
* field set to "@om-field-type".
|
||||
*/
|
||||
public static List<Type> getTypes(String jsonSchemaFile) throws IOException {
|
||||
public static List<Type> getFieldTypes(String jsonSchemaFile) throws IOException {
|
||||
JsonNode node =
|
||||
OBJECT_MAPPER.readTree(
|
||||
Objects.requireNonNull(JsonUtils.class.getClassLoader().getResourceAsStream(jsonSchemaFile)));
|
||||
@ -236,12 +268,15 @@ public final class JsonUtils {
|
||||
Iterator<Entry<String, JsonNode>> definitions = node.get("definitions").fields();
|
||||
while (definitions != null && definitions.hasNext()) {
|
||||
Entry<String, JsonNode> entry = definitions.next();
|
||||
String typeName = entry.getKey();
|
||||
JsonNode value = entry.getValue();
|
||||
if (JsonUtils.hasAnnotation(value, JsonUtils.TYPE_ANNOTATION)) {
|
||||
if (JsonUtils.hasAnnotation(value, JsonUtils.FIELD_TYPE_ANNOTATION)) {
|
||||
String description = String.valueOf(value.get("description"));
|
||||
Type type =
|
||||
new Type()
|
||||
.withName(entry.getKey())
|
||||
.withName(typeName)
|
||||
.withCategory(Category.Field)
|
||||
.withFullyQualifiedName(typeName)
|
||||
.withNameSpace(jsonNamespace)
|
||||
.withDescription(description)
|
||||
.withDisplayName(entry.getKey())
|
||||
@ -251,4 +286,33 @@ public final class JsonUtils {
|
||||
}
|
||||
return types;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all the fields types from the `definitions` section of a JSON schema file that are annotated with "$comment"
|
||||
* field set to "@om-entity-type".
|
||||
*/
|
||||
public static Type getEntityType(String jsonSchemaFile) throws IOException {
|
||||
JsonNode node =
|
||||
OBJECT_MAPPER.readTree(
|
||||
Objects.requireNonNull(JsonUtils.class.getClassLoader().getResourceAsStream(jsonSchemaFile)));
|
||||
if (!JsonUtils.hasAnnotation(node, JsonUtils.ENTITY_TYPE_ANNOTATION)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String fileName = Paths.get(jsonSchemaFile).getFileName().toString();
|
||||
String entityName = fileName.replace(" ", "").replace(".json", "");
|
||||
|
||||
String namespaceFile = Paths.get(jsonSchemaFile).getParent().getFileName().toString();
|
||||
String namespace = namespaceFile.replace(" ", "").replace(".json", "");
|
||||
|
||||
String description = String.valueOf(node.get("description"));
|
||||
return new Type()
|
||||
.withName(entityName)
|
||||
.withCategory(Category.Entity)
|
||||
.withFullyQualifiedName(entityName)
|
||||
.withNameSpace(namespace)
|
||||
.withDescription(description)
|
||||
.withDisplayName(entityName)
|
||||
.withSchema(node.toPrettyString());
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
{
|
||||
"$id": "https://open-metadata.org/schema/api/data/createAttribute.json",
|
||||
"$id": "https://open-metadata.org/schema/api/data/createType.json",
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "createType",
|
||||
"description": "Create a Type to be used for extending entities.",
|
||||
@ -7,7 +7,7 @@
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "Unique name that identifies a Type.",
|
||||
"$ref": "../type/basic.json#/definitions/entityName"
|
||||
"$ref": "../entity/type.json#/definitions/typeName"
|
||||
},
|
||||
"nameSpace": {
|
||||
"description": "Namespace or group to which this type belongs to.",
|
||||
@ -20,7 +20,10 @@
|
||||
},
|
||||
"description": {
|
||||
"description": "Optional description of the type.",
|
||||
"type": "string"
|
||||
"$ref" : "../type/basic.json#/definitions/markdown"
|
||||
},
|
||||
"category" : {
|
||||
"$ref" : "../entity/type.json#/definitions/category"
|
||||
},
|
||||
"schema": {
|
||||
"description": "JSON schema encoded as string. This will be used to validate the type values.",
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
"$id": "https://open-metadata.org/schema/entity/data/table.json",
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Table",
|
||||
"$comment" : "@om-entity-type",
|
||||
"description": "This schema defines the Table entity. A Table organizes data in rows and columns and is defined by a Schema. OpenMetadata does not have a separate abstraction for Schema. Both Table and Schema are captured in this entity.",
|
||||
"type": "object",
|
||||
"javaType": "org.openmetadata.catalog.entity.data.Table",
|
||||
|
||||
@ -2,10 +2,63 @@
|
||||
"$id": "https://open-metadata.org/schema/entity/type.json",
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Type",
|
||||
"description": "This schema defines a type entity used for extending an entity with custom attributes.",
|
||||
"description": "This schema defines a type as an entity. Types includes field types and entity types. Custom types can also be defined by the users to extend the metadata system.",
|
||||
"type": "object",
|
||||
"javaType": "org.openmetadata.catalog.entity.Type",
|
||||
"javaInterfaces": ["org.openmetadata.catalog.EntityInterface"],
|
||||
"definitions": {
|
||||
"category" : {
|
||||
"description" : "Metadata category to which a type belongs to.",
|
||||
"type": "string",
|
||||
"javaType": "org.openmetadata.catalog.entity.type.Category",
|
||||
"enum": [
|
||||
"field",
|
||||
"entity"
|
||||
],
|
||||
"javaEnums": [
|
||||
{
|
||||
"name": "Field"
|
||||
},
|
||||
{
|
||||
"name": "Entity"
|
||||
}
|
||||
]
|
||||
},
|
||||
"fieldName": {
|
||||
"description": "Name of the entity field. Note a field name must be unique for an entity. Field name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.",
|
||||
"type" : "string",
|
||||
"pattern": "^[a-z][a-zA-Z0-9]+$"
|
||||
},
|
||||
"typeName": {
|
||||
"description": "Name of the field or entity types. Note a field name must be unique for an entity. Field name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.",
|
||||
"type" : "string",
|
||||
"pattern": "^[a-z][a-zA-Z0-9]+$"
|
||||
},
|
||||
"customField" : {
|
||||
"description" : "Type used for adding custom field to an entity to extend it.",
|
||||
"type" : "object",
|
||||
"javaType": "org.openmetadata.catalog.entity.type.CustomField",
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "Name of the entity field. Note a field name must be unique for an entity. Field name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.",
|
||||
"$ref" : "#/definitions/fieldName"
|
||||
},
|
||||
"description": {
|
||||
"$ref" : "../type/basic.json#/definitions/markdown"
|
||||
},
|
||||
"fieldType": {
|
||||
"description": "Reference to a field type. Only field types are allows and entity types are not allowed as custom fields to extend an existing entity",
|
||||
"$ref": "../type/entityReference.json"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"description",
|
||||
"fieldType"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"properties": {
|
||||
"id": {
|
||||
"description": "Unique identifier of the type instance.",
|
||||
@ -13,29 +66,39 @@
|
||||
},
|
||||
"name": {
|
||||
"description": "Unique name that identifies the type.",
|
||||
"$ref": "../type/basic.json#/definitions/entityName"
|
||||
"$ref": "#/definitions/typeName"
|
||||
},
|
||||
"fullyQualifiedName": {
|
||||
"description": "FullyQualifiedName same as `name`.",
|
||||
"$ref": "../type/basic.json#/definitions/fullyQualifiedEntityName"
|
||||
},
|
||||
"nameSpace": {
|
||||
"description": "Namespace or group to which this type belongs to.",
|
||||
"type": "string",
|
||||
"default" : "custom"
|
||||
},
|
||||
"displayName": {
|
||||
"description": "Display Name that identifies this type.",
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"description": "Optional description of entity.",
|
||||
"type": "string"
|
||||
"$ref" : "../type/basic.json#/definitions/markdown"
|
||||
},
|
||||
"nameSpace": {
|
||||
"description": "Namespace or group to which this type belongs to. For example, some of the field types commonly used can come from `basic` namespace. Some of the entities such as `table`, `database`, etc. come from `data` namespace.",
|
||||
"type": "string",
|
||||
"default" : "custom"
|
||||
},
|
||||
"category" : {
|
||||
"$ref" : "#/definitions/category"
|
||||
},
|
||||
"schema": {
|
||||
"description": "JSON schema encoded as string that defines the type. This will be used to validate the type values.",
|
||||
"$ref": "../type/basic.json#/definitions/jsonSchema"
|
||||
},
|
||||
"customFields" : {
|
||||
"description": "Custom fields added to extend the entity. Only available for entity type",
|
||||
"type" : "array",
|
||||
"items" : {
|
||||
"$ref" : "#/definitions/customField"
|
||||
}
|
||||
},
|
||||
"version": {
|
||||
"description": "Metadata version of the entity.",
|
||||
"$ref": "../type/entityHistory.json#/definitions/entityVersion"
|
||||
@ -61,7 +124,7 @@
|
||||
"name",
|
||||
"nameSpace",
|
||||
"description",
|
||||
"type"
|
||||
"schema"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
|
||||
@ -5,17 +5,17 @@
|
||||
"description": "This schema defines basic common types that are used by other schemas.",
|
||||
"definitions": {
|
||||
"integer" : {
|
||||
"$comment" : "@om-type",
|
||||
"$comment" : "@om-field-type",
|
||||
"description": "An integer type.",
|
||||
"type" : "integer"
|
||||
},
|
||||
"number" : {
|
||||
"$comment" : "@om-type",
|
||||
"$comment" : "@om-field-type",
|
||||
"description": "A numeric type that includes integer or floating point numbers.",
|
||||
"type" : "integer"
|
||||
},
|
||||
"string" : {
|
||||
"$comment" : "@om-type",
|
||||
"$comment" : "@om-field-type",
|
||||
"description": "A String type.",
|
||||
"type" : "string"
|
||||
},
|
||||
@ -25,7 +25,7 @@
|
||||
"format": "uuid"
|
||||
},
|
||||
"email": {
|
||||
"$comment" : "@om-type",
|
||||
"$comment" : "@om-field-type",
|
||||
"description": "Email address of a user or other entities.",
|
||||
"type": "string",
|
||||
"format": "email",
|
||||
@ -34,7 +34,7 @@
|
||||
"maxLength": 127
|
||||
},
|
||||
"timestamp": {
|
||||
"$comment" : "@om-type",
|
||||
"$comment" : "@om-field-type",
|
||||
"description": "Timestamp in Unix epoch time milliseconds.",
|
||||
"@comment": "Note that during code generation this is converted into long",
|
||||
"type": "integer",
|
||||
@ -46,7 +46,7 @@
|
||||
"format": "uri"
|
||||
},
|
||||
"timeInterval": {
|
||||
"$comment" : "@om-type",
|
||||
"$comment" : "@om-field-type",
|
||||
"type": "object",
|
||||
"description": "Time interval in unixTimeMillis.",
|
||||
"javaType": "org.openmetadata.catalog.type.TimeInterval",
|
||||
@ -63,18 +63,18 @@
|
||||
"additionalProperties": false
|
||||
},
|
||||
"duration": {
|
||||
"$comment" : "@om-type",
|
||||
"$comment" : "@om-field-type",
|
||||
"description": "Duration in ISO 8601 format in UTC. Example - 'P23DT23H'.",
|
||||
"type": "string"
|
||||
},
|
||||
"date": {
|
||||
"$comment" : "@om-type",
|
||||
"$comment" : "@om-field-type",
|
||||
"description": "Date in ISO 8601 format in UTC. Example - '2018-11-13'.",
|
||||
"type": "string",
|
||||
"format": "date"
|
||||
},
|
||||
"dateTime": {
|
||||
"$comment" : "@om-type",
|
||||
"$comment" : "@om-field-type",
|
||||
"description": "Date and time in ISO 8601 format. Example - '2018-11-13T20:20:39+00:00'.",
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
@ -97,12 +97,12 @@
|
||||
"maxLength": 256
|
||||
},
|
||||
"sqlQuery": {
|
||||
"$comment" : "@om-type",
|
||||
"$comment" : "@om-field-type",
|
||||
"description": "SQL query statement. Example - 'select * from orders'.",
|
||||
"type": "string"
|
||||
},
|
||||
"markdown": {
|
||||
"$comment" : "@om-type",
|
||||
"$comment" : "@om-field-type",
|
||||
"description": "Text in Markdown format",
|
||||
"type": "string"
|
||||
},
|
||||
|
||||
@ -49,6 +49,7 @@ public abstract class CatalogApplicationTest {
|
||||
// The system properties are provided by maven-surefire for testing with mysql and postgres
|
||||
final String jdbcContainerClassName = System.getProperty("jdbcContainerClassName");
|
||||
final String jdbcContainerImage = System.getProperty("jdbcContainerImage");
|
||||
LOG.info("Using test container class {} and image {}", jdbcContainerClassName, jdbcContainerImage);
|
||||
|
||||
SQL_CONTAINER =
|
||||
(JdbcDatabaseContainer<?>)
|
||||
|
||||
@ -145,6 +145,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K> extends C
|
||||
protected boolean supportsAuthorizedMetadataOperations = true;
|
||||
protected boolean supportsFieldsQueryParam = true;
|
||||
protected boolean supportsEmptyDescription = true;
|
||||
protected boolean supportsNameWithDot = true;
|
||||
|
||||
public static final String DATA_STEWARD_ROLE_NAME = "DataSteward";
|
||||
public static final String DATA_CONSUMER_ROLE_NAME = "DataConsumer";
|
||||
@ -623,14 +624,14 @@ public abstract class EntityResourceTest<T extends EntityInterface, K> extends C
|
||||
// Common entity tests for POST operations
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@Test
|
||||
void post_entityCreateWithInvalidName_400() {
|
||||
protected void post_entityCreateWithInvalidName_400() {
|
||||
// Create an entity with mandatory name field null
|
||||
final K request = createRequest(null, "description", "displayName", null);
|
||||
assertResponse(() -> createEntity(request, ADMIN_AUTH_HEADERS), BAD_REQUEST, "[name must not be null]");
|
||||
|
||||
// Create an entity with mandatory name field empty
|
||||
final K request1 = createRequest("", "description", "displayName", null);
|
||||
assertResponse(() -> createEntity(request1, ADMIN_AUTH_HEADERS), BAD_REQUEST, ENTITY_NAME_LENGTH_ERROR);
|
||||
assertResponseContains(() -> createEntity(request1, ADMIN_AUTH_HEADERS), BAD_REQUEST, ENTITY_NAME_LENGTH_ERROR);
|
||||
|
||||
// Create an entity with mandatory name field too long
|
||||
final K request2 = createRequest(LONG_ENTITY_NAME, "description", "displayName", null);
|
||||
@ -698,6 +699,9 @@ public abstract class EntityResourceTest<T extends EntityInterface, K> extends C
|
||||
|
||||
@Test
|
||||
void post_entityWithDots_200() throws HttpResponseException {
|
||||
if (!supportsNameWithDot) {
|
||||
return;
|
||||
}
|
||||
// Entity without "." should not have quoted fullyQualifiedName
|
||||
String name = format("%s_foo_bar", entityType);
|
||||
K request = createRequest(name, "", null, null);
|
||||
|
||||
@ -14,20 +14,30 @@
|
||||
package org.openmetadata.catalog.resources.metadata;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
|
||||
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
|
||||
import static org.openmetadata.catalog.util.TestUtils.assertResponseContains;
|
||||
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import javax.ws.rs.client.WebTarget;
|
||||
import javax.ws.rs.core.Response.Status;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.HttpResponseException;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.MethodOrderer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInfo;
|
||||
import org.junit.jupiter.api.TestMethodOrder;
|
||||
import org.openmetadata.catalog.Entity;
|
||||
import org.openmetadata.catalog.api.CreateType;
|
||||
import org.openmetadata.catalog.entity.Type;
|
||||
import org.openmetadata.catalog.entity.type.Category;
|
||||
import org.openmetadata.catalog.entity.type.CustomField;
|
||||
import org.openmetadata.catalog.resources.EntityResourceTest;
|
||||
import org.openmetadata.catalog.resources.types.TypeResource;
|
||||
import org.openmetadata.catalog.resources.types.TypeResource.TypeList;
|
||||
@ -43,12 +53,63 @@ public class TypeResourceTest extends EntityResourceTest<Type, CreateType> {
|
||||
super(Entity.TYPE, Type.class, TypeList.class, "metadata/types", TypeResource.FIELDS);
|
||||
supportsEmptyDescription = false;
|
||||
supportsFieldsQueryParam = false;
|
||||
supportsNameWithDot = false;
|
||||
}
|
||||
|
||||
@BeforeAll
|
||||
public void setup(TestInfo test) throws IOException, URISyntaxException {
|
||||
super.setup(test);
|
||||
INT_TYPE = getEntityByName("type.basic.integer", "", ADMIN_AUTH_HEADERS);
|
||||
INT_TYPE = getEntityByName("integer", "", ADMIN_AUTH_HEADERS);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void post_entityCreateWithInvalidName_400() {
|
||||
String[][] tests = {
|
||||
{"Abcd", "[name must match \"^[a-z][a-zA-Z0-9]+$\"]"},
|
||||
{"a bc", "[name must match \"^[a-z][a-zA-Z0-9]+$\"]"}, // Name must not have space
|
||||
{"a_bc", "[name must match \"^[a-z][a-zA-Z0-9]+$\"]"}, // Name must not be underscored
|
||||
{"a-bc", "[name must match \"^[a-z][a-zA-Z0-9]+$\"]"}, // Name must not be hyphened
|
||||
};
|
||||
|
||||
CreateType create = createRequest("placeHolder", "", "", null);
|
||||
for (String[] test : tests) {
|
||||
LOG.info("Testing with the name {}", test[0]);
|
||||
create.withName(test[0]);
|
||||
assertResponseContains(() -> createEntity(create, ADMIN_AUTH_HEADERS), Status.BAD_REQUEST, test[1]);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void put_customField_200() throws HttpResponseException {
|
||||
Type tableEntity = getEntityByName("table", "customFields", ADMIN_AUTH_HEADERS);
|
||||
assertTrue(listOrEmpty(tableEntity.getCustomFields()).isEmpty());
|
||||
|
||||
// Add a custom field with name intA with type integer
|
||||
CustomField fieldA =
|
||||
new CustomField().withName("intA").withDescription("intA").withFieldType(INT_TYPE.getEntityReference());
|
||||
tableEntity = addCustomField(tableEntity.getId(), fieldA, Status.OK, ADMIN_AUTH_HEADERS);
|
||||
assertEquals(1, tableEntity.getCustomFields().size());
|
||||
assertEquals(fieldA, tableEntity.getCustomFields().get(0));
|
||||
|
||||
// Add a second field with name intB with type integer
|
||||
CustomField fieldB =
|
||||
new CustomField().withName("intB").withDescription("intB").withFieldType(INT_TYPE.getEntityReference());
|
||||
tableEntity = addCustomField(tableEntity.getId(), fieldB, Status.OK, ADMIN_AUTH_HEADERS);
|
||||
assertEquals(2, tableEntity.getCustomFields().size());
|
||||
assertEquals(fieldA, tableEntity.getCustomFields().get(0));
|
||||
assertEquals(fieldB, tableEntity.getCustomFields().get(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void put_customFieldToFieldType_4xx() {
|
||||
// Adding a custom field to a field type is not allowed (only entity type is allowed)
|
||||
CustomField field =
|
||||
new CustomField().withName("intA").withDescription("intA").withFieldType(INT_TYPE.getEntityReference());
|
||||
assertResponse(
|
||||
() -> addCustomField(INT_TYPE.getId(), field, Status.CREATED, ADMIN_AUTH_HEADERS),
|
||||
Status.BAD_REQUEST,
|
||||
"Field types can't be extended");
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -61,12 +122,23 @@ public class TypeResourceTest extends EntityResourceTest<Type, CreateType> {
|
||||
return type;
|
||||
}
|
||||
|
||||
public Type addCustomField(UUID entityTypeId, CustomField customField, Status status, Map<String, String> authHeaders)
|
||||
throws HttpResponseException {
|
||||
WebTarget target = getResource(entityTypeId);
|
||||
return TestUtils.put(target, customField, Type.class, status, authHeaders);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CreateType createRequest(String name, String description, String displayName, EntityReference owner) {
|
||||
if (name != null) {
|
||||
name = name.replaceAll("[. _-]", "");
|
||||
}
|
||||
System.out.println("XXX Using the name " + name);
|
||||
return new CreateType()
|
||||
.withName(name)
|
||||
.withDescription(description)
|
||||
.withDisplayName(displayName)
|
||||
.withCategory(Category.Field)
|
||||
.withSchema(INT_TYPE.getSchema());
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user