yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-26 06:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #4921 Add support for type system (#4922)

Browse Source
This commit is contained in:
Suresh Srinivas 2022-05-14 10:57:19 -07:00 committed by GitHub
parent de16e7547b
commit 6ce528d70a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 1091 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
bootstrap/sql/com.mysql.cj.jdbc.Driver/v002__create_db_connection_info.sql Normal file
View File

@ -0,0 +1,19 @@
--
-- Table to be used for generic entities that are limited in number. Examples of generic entities are
-- Attribute entity, domain entities etc.
--
-- This reduces need for defining a table per entity.
--
CREATE TABLE IF NOT EXISTS generic_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
-- Fully qualified name formed by entityType + "." + entityName
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
ALTER TABLE webhook_entity
DROP COLUMN deleted;

19
bootstrap/sql/org.postgresql.Driver/v002__create_db_connection_info.sql Normal file
View File

@ -0,0 +1,19 @@
--
-- Table to be used for generic entities that are limited in number. Examples of generic entities are
-- Attribute entity, domain entities etc.
--
-- This reduces need for defining a table per entity.
--
CREATE TABLE IF NOT EXISTS generic_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
-- Fully qualified name formed by entityType + "." + entityName
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
ALTER TABLE webhook_entity
DROP COLUMN deleted;

6
catalog-rest-service/pom.xml
View File

@ -306,6 +306,12 @@
<version>20220320</version>
</dependency>
<dependency>
<groupId>com.networknt</groupId>
<artifactId>json-schema-validator</artifactId>
<version>1.0.69</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.jsonwebtoken/jjwt -->
<dependency>
<groupId>io.jsonwebtoken</groupId>

37
catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java
View File

@ -64,7 +64,7 @@ public final class Entity {
public static final String FIELD_DISPLAY_NAME = "displayName";
//
// Services
// Service entities
//
public static final String DATABASE_SERVICE = "databaseService";
public static final String MESSAGING_SERVICE = "messagingService";
@ -73,7 +73,7 @@ public final class Entity {
public static final String STORAGE_SERVICE = "storageService";
//
// Data assets
// Data asset entities
//
public static final String TABLE = "table";
public static final String DATABASE = "database";
@ -94,21 +94,22 @@ public final class Entity {
public static final String GLOSSARY_TERM = "glossaryTerm";
public static final String TAG = "tag";
public static final String TAG_CATEGORY = "tagCategory";
public static final String TYPE = "type";
//
// Policies
// Policy entity
//
public static final String POLICY = "policy";
//
// Role, team and user
// Role, team and user entities
//
public static final String ROLE = "role";
public static final String USER = "user";
public static final String TEAM = "team";
//
// Operations
// Operation related entities
//
public static final String INGESTION_PIPELINE = "ingestionPipeline";
public static final String WEBHOOK = "webhook";
@ -145,8 +146,15 @@ public final class Entity {
entityRepository.getClass().getSimpleName());
}
public static void validateEntity(String entityType) {
String canonicalEntity = CANONICAL_ENTITY_NAME_MAP.get(entityType.toLowerCase());
if (canonicalEntity == null) {
throw new IllegalArgumentException(CatalogExceptionMessage.invalidEntity(entityType));
}
}
public static EntityReference getEntityReference(EntityReference ref) throws IOException {
return ref == null ? null : getEntityReferenceById(ref.getType(), ref.getId());
return ref == null ? null : getEntityReferenceById(ref.getType(), ref.getId(), Include.NON_DELETED);
}
public static <T> EntityReference getEntityReference(T entity) {
@ -154,23 +162,14 @@ public final class Entity {
return getEntityRepository(entityType).getEntityInterface(entity).getEntityReference();
}
public static EntityReference getEntityReferenceById(@NonNull String entityType, @NonNull UUID id)
throws IOException {
return getEntityReferenceById(entityType, id, Include.NON_DELETED);
}
public static EntityReference getEntityReferenceById(@NonNull String entityType, @NonNull UUID id, Include include)
throws IOException {
EntityDAO<?> dao = DAO_MAP.get(entityType);
if (dao == null) {
EntityRepository<?> repository = ENTITY_REPOSITORY_MAP.get(entityType);
if (repository == null) {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(entityType));
}
return dao.findEntityReferenceById(id, include);
}
public static EntityReference getEntityReferenceByName(@NonNull String entityType, @NonNull String fqn)
throws IOException {
return getEntityReferenceByName(entityType, fqn, Include.NON_DELETED);
include = repository.supportsSoftDelete ? Include.ALL : include;
return repository.dao.findEntityReferenceById(id, include);
}
public static EntityReference getEntityReferenceByName(

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogExceptionMessage.java
View File

@ -119,4 +119,8 @@ public final class CatalogExceptionMessage {
public static String entityIsNotEmpty(String entityType) {
return String.format("%s is not empty", entityType);
}
public static String invalidEntity(String entity) {
return String.format("Invalid entity %s", entity);
}
}

37
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java
View File

@ -33,6 +33,7 @@ import org.jdbi.v3.sqlobject.statement.SqlQuery;
import org.jdbi.v3.sqlobject.statement.SqlUpdate;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.Bot;
import org.openmetadata.catalog.entity.Type;
import org.openmetadata.catalog.entity.data.Chart;
import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.entity.data.Database;
@ -81,6 +82,7 @@ import org.openmetadata.catalog.jdbi3.StorageServiceRepository.StorageServiceEnt
import org.openmetadata.catalog.jdbi3.TableRepository.TableEntityInterface;
import org.openmetadata.catalog.jdbi3.TeamRepository.TeamEntityInterface;
import org.openmetadata.catalog.jdbi3.TopicRepository.TopicEntityInterface;
import org.openmetadata.catalog.jdbi3.TypeRepository.TypeEntityInterface;
import org.openmetadata.catalog.jdbi3.UserRepository.UserEntityInterface;
import org.openmetadata.catalog.jdbi3.WebhookRepository.WebhookEntityInterface;
import org.openmetadata.catalog.jdbi3.locator.ConnectionAwareSqlQuery;
@ -199,6 +201,9 @@ public interface CollectionDAO {
@CreateSqlObject
WebhookDAO webhookDAO();
@CreateSqlObject
GenericEntityDAO genericEntityDAO();
interface DashboardDAO extends EntityDAO<Dashboard> {
@Override
default String getTableName() {
@ -1248,6 +1253,11 @@ public interface CollectionDAO {
default EntityReference getEntityReference(Webhook entity) {
return new WebhookEntityInterface(entity).getEntityReference();
}
@Override
default boolean supportsSoftDelete() {
return false;
}
}
interface TagCategoryDAO extends EntityDAO<TagCategory> {
@ -1710,4 +1720,31 @@ public interface CollectionDAO {
+ "ORDER BY eventTime ASC")
List<String> listWithoutEntityFilter(@Bind("eventType") String eventType, @Bind("timestamp") long timestamp);
}
interface GenericEntityDAO extends EntityDAO<Type> {
@Override
default String getTableName() {
return "generic_entity";
}
@Override
default Class<Type> getEntityClass() {
return Type.class;
}
@Override
default String getNameColumn() {
return "fullyQualifiedName";
}
@Override
default EntityReference getEntityReference(Type entity) {
return new TypeEntityInterface(entity).getEntityReference();
}
@Override
default boolean supportsSoftDelete() {
return false;
}
}
}

8
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityDAO.java
View File

@ -45,6 +45,10 @@ public interface EntityDAO<T> {
EntityReference getEntityReference(T entity);
default boolean supportsSoftDelete() {
return true;
}
/** Common queries for all entities implemented here. Do not override. */
@ConnectionAwareSqlUpdate(value = "INSERT INTO <table> (json) VALUES (:json)", connectionType = MYSQL)
@ConnectionAwareSqlUpdate(value = "INSERT INTO <table> (json) VALUES (:json :: jsonb)", connectionType = POSTGRES)
@ -113,6 +117,10 @@ public interface EntityDAO<T> {
}
default String getCondition(Include include) {
if (!supportsSoftDelete()) {
return "";
}
if (include == null || include == Include.NON_DELETED) {
return "AND deleted = FALSE";
}

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java
View File

@ -115,10 +115,10 @@ public abstract class EntityRepository<T> {
private final String collectionPath;
private final Class<T> entityClass;
private final String entityType;
protected final EntityDAO<T> dao;
public final EntityDAO<T> dao;
protected final CollectionDAO daoCollection;
protected final List<String> allowedFields;
protected boolean supportsSoftDelete = true;
public final boolean supportsSoftDelete;
protected final boolean supportsTags;
protected final boolean supportsOwner;
protected final boolean supportsFollower;
@ -149,6 +149,7 @@ public abstract class EntityRepository<T> {
this.supportsTags = allowedFields.contains(FIELD_TAGS);
this.supportsOwner = allowedFields.contains(FIELD_OWNER);
this.supportsSoftDelete = allowedFields.contains(FIELD_DELETED);
this.supportsFollower = allowedFields.contains(FIELD_FOLLOWERS);
Entity.registerEntity(entityClass, entityType, dao, this);
}

15
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LineageRepository.java
View File

@ -24,6 +24,7 @@ import org.openmetadata.catalog.api.lineage.AddLineage;
import org.openmetadata.catalog.type.Edge;
import org.openmetadata.catalog.type.EntityLineage;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.Include;
import org.openmetadata.catalog.type.Relationship;
public class LineageRepository {
@ -35,14 +36,14 @@ public class LineageRepository {
@Transaction
public EntityLineage get(String entityType, String id, int upstreamDepth, int downstreamDepth) throws IOException {
EntityReference ref = Entity.getEntityReferenceById(entityType, UUID.fromString(id));
EntityReference ref = Entity.getEntityReferenceById(entityType, UUID.fromString(id), Include.NON_DELETED);
return getLineage(ref, upstreamDepth, downstreamDepth);
}
@Transaction
public EntityLineage getByName(String entityType, String fqn, int upstreamDepth, int downstreamDepth)
throws IOException {
EntityReference ref = Entity.getEntityReferenceByName(entityType, fqn);
EntityReference ref = Entity.getEntityReferenceByName(entityType, fqn, Include.NON_DELETED);
return getLineage(ref, upstreamDepth, downstreamDepth);
}
@ -50,11 +51,11 @@ public class LineageRepository {
public void addLineage(AddLineage addLineage) throws IOException {
// Validate from entity
EntityReference from = addLineage.getEdge().getFromEntity();
from = Entity.getEntityReferenceById(from.getType(), from.getId());
from = Entity.getEntityReferenceById(from.getType(), from.getId(), Include.NON_DELETED);
// Validate to entity
EntityReference to = addLineage.getEdge().getToEntity();
to = Entity.getEntityReferenceById(to.getType(), to.getId());
to = Entity.getEntityReferenceById(to.getType(), to.getId(), Include.NON_DELETED);
// Finally, add lineage relationship
dao.relationshipDAO()
@ -64,10 +65,10 @@ public class LineageRepository {
@Transaction
public boolean deleteLineage(String fromEntity, String fromId, String toEntity, String toId) throws IOException {
// Validate from entity
EntityReference from = Entity.getEntityReferenceById(fromEntity, UUID.fromString(fromId));
EntityReference from = Entity.getEntityReferenceById(fromEntity, UUID.fromString(fromId), Include.NON_DELETED);
// Validate to entity
EntityReference to = Entity.getEntityReferenceById(toEntity, UUID.fromString(toId));
EntityReference to = Entity.getEntityReferenceById(toEntity, UUID.fromString(toId), Include.NON_DELETED);
// Finally, delete lineage relationship
return dao.relationshipDAO()
@ -97,7 +98,7 @@ public class LineageRepository {
// Add entityReference details
for (int i = 0; i < lineage.getNodes().size(); i++) {
EntityReference ref = lineage.getNodes().get(i);
ref = Entity.getEntityReferenceById(ref.getType(), ref.getId());
ref = Entity.getEntityReferenceById(ref.getType(), ref.getId(), Include.NON_DELETED);
lineage.getNodes().set(i, ref);
}
return lineage;

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java
View File

@ -35,6 +35,7 @@ import org.openmetadata.catalog.entity.data.MlModel;
import org.openmetadata.catalog.resources.mlmodels.MlModelResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.Include;
import org.openmetadata.catalog.type.MlFeature;
import org.openmetadata.catalog.type.MlFeatureSource;
import org.openmetadata.catalog.type.MlHyperParameter;
@ -125,7 +126,8 @@ public class MlModelRepository extends EntityRepository<MlModel> {
private void validateMlDataSource(MlFeatureSource source) throws IOException {
if (source.getDataSource() != null) {
Entity.getEntityReferenceById(source.getDataSource().getType(), source.getDataSource().getId());
Entity.getEntityReferenceById(
source.getDataSource().getType(), source.getDataSource().getId(), Include.NON_DELETED);
}
}

203
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TypeRepository.java Normal file
View File

@ -0,0 +1,203 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import java.io.IOException;
import java.net.URI;
import java.util.UUID;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.Type;
import org.openmetadata.catalog.resources.types.TypeResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.FullyQualifiedName;
public class TypeRepository extends EntityRepository<Type> {
// TODO fix this
private static final String UPDATE_FIELDS = "";
private static final String PATCH_FIELDS = "";
public TypeRepository(CollectionDAO dao) {
super(
TypeResource.COLLECTION_PATH,
Entity.TYPE,
Type.class,
dao.genericEntityDAO(),
dao,
PATCH_FIELDS,
UPDATE_FIELDS);
allowEdits = true;
}
@Override
public Type setFields(Type attribute, Fields fields) throws IOException {
return attribute;
}
@Override
public void prepare(Type type) throws IOException {
type.setFullyQualifiedName(getEntityInterface(type).getFullyQualifiedName());
}
@Override
public void storeEntity(Type type, boolean update) throws IOException {
URI href = type.getHref();
type.withHref(null);
store(type.getId(), type, update);
type.withHref(href);
}
@Override
public void storeRelationships(Type type) {
// Nothing to do
}
@Override
public EntityInterface<Type> getEntityInterface(Type entity) {
return new TypeEntityInterface(entity);
}
@Override
public EntityUpdater getUpdater(Type original, Type updated, Operation operation) {
return new AttributeUpdater(original, updated, operation);
}
public static class TypeEntityInterface extends EntityInterface<Type> {
public TypeEntityInterface(Type entity) {
super(Entity.TYPE, entity);
}
@Override
public UUID getId() {
return entity.getId();
}
@Override
public String getDescription() {
return entity.getDescription();
}
@Override
public String getDisplayName() {
return entity.getDisplayName();
}
@Override
public String getName() {
return entity.getName();
}
@Override
public Boolean isDeleted() {
return false;
}
@Override
public EntityReference getOwner() {
return null;
}
@Override
public String getFullyQualifiedName() {
return FullyQualifiedName.build(entityType, entity.getNameSpace(), entity.getName());
}
@Override
public Double getVersion() {
return entity.getVersion();
}
@Override
public String getUpdatedBy() {
return entity.getUpdatedBy();
}
@Override
public long getUpdatedAt() {
return entity.getUpdatedAt();
}
@Override
public URI getHref() {
return entity.getHref();
}
@Override
public ChangeDescription getChangeDescription() {
return entity.getChangeDescription();
}
@Override
public Type getEntity() {
return entity;
}
@Override
public EntityReference getContainer() {
return null;
}
@Override
public void setId(UUID id) {
entity.setId(id);
}
@Override
public void setDescription(String description) {
entity.setDescription(description);
}
@Override
public void setDisplayName(String displayName) {
entity.setDisplayName(displayName);
}
@Override
public void setName(String name) {
entity.setName(name);
}
@Override
public void setUpdateDetails(String updatedBy, long updatedAt) {
entity.setUpdatedBy(updatedBy);
entity.setUpdatedAt(updatedAt);
}
@Override
public void setChangeDescription(Double newVersion, ChangeDescription changeDescription) {
entity.setVersion(newVersion);
entity.setChangeDescription(changeDescription);
}
@Override
public void setDeleted(boolean flag) {}
@Override
public Type withHref(URI href) {
return entity.withHref(href);
}
}
/** Handles entity updated from PUT and POST operation. */
public class AttributeUpdater extends EntityUpdater {
public AttributeUpdater(Type original, Type updated, Operation operation) {
super(original, updated, operation);
}
}
}

8
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UsageRepository.java
View File

@ -42,14 +42,14 @@ public class UsageRepository {
@Transaction
public EntityUsage get(String entityType, String id, String date, int days) throws IOException {
EntityReference ref = Entity.getEntityReferenceById(entityType, UUID.fromString(id));
EntityReference ref = Entity.getEntityReferenceById(entityType, UUID.fromString(id), Include.NON_DELETED);
List<UsageDetails> usageDetails = dao.usageDAO().getUsageById(id, date, days - 1);
return new EntityUsage().withUsage(usageDetails).withEntity(ref);
}
@Transaction
public EntityUsage getByName(String entityType, String fqn, String date, int days) throws IOException {
EntityReference ref = Entity.getEntityReferenceByName(entityType, fqn);
EntityReference ref = Entity.getEntityReferenceByName(entityType, fqn, Include.NON_DELETED);
List<UsageDetails> usageDetails = dao.usageDAO().getUsageById(ref.getId().toString(), date, days - 1);
return new EntityUsage().withUsage(usageDetails).withEntity(ref);
}
@ -57,13 +57,13 @@ public class UsageRepository {
@Transaction
public void create(String entityType, String id, DailyCount usage) throws IOException {
// Validate data entity for which usage is being collected
Entity.getEntityReferenceById(entityType, UUID.fromString(id));
Entity.getEntityReferenceById(entityType, UUID.fromString(id), Include.NON_DELETED);
addUsage(entityType, id, usage);
}
@Transaction
public void createByName(String entityType, String fullyQualifiedName, DailyCount usage) throws IOException {
EntityReference ref = Entity.getEntityReferenceByName(entityType, fullyQualifiedName);
EntityReference ref = Entity.getEntityReferenceByName(entityType, fullyQualifiedName, Include.NON_DELETED);
addUsage(entityType, ref.getId().toString(), usage);
LOG.info("Usage successfully posted by name");
}

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/WebhookRepository.java
View File

@ -184,7 +184,7 @@ public class WebhookRepository extends EntityRepository<Webhook> {
@Override
public Boolean isDeleted() {
return entity.getDeleted();
return false;
}
@Override
@ -256,7 +256,7 @@ public class WebhookRepository extends EntityRepository<Webhook> {
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
/* soft-delete not supported */
}
@Override

14
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java
View File

@ -56,8 +56,6 @@ import org.openmetadata.catalog.type.Include;
import org.openmetadata.catalog.type.Webhook;
import org.openmetadata.catalog.type.Webhook.Status;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.RestUtil;
import org.openmetadata.catalog.util.ResultList;
@Path("/v1/webhook")
@ -120,16 +118,8 @@ public class WebhookResource extends EntityResource<Webhook, WebhookRepository>
@DefaultValue("non-deleted")
Include include)
throws IOException {
RestUtil.validateCursors(before, after);
ListFilter filter = new ListFilter(include);
ResultList<Webhook> webhooks;
if (before != null) { // Reverse paging
webhooks = dao.listBefore(uriInfo, Fields.EMPTY_FIELDS, filter, limitParam, before);
} else { // Forward paging or first page
webhooks = dao.listAfter(uriInfo, Fields.EMPTY_FIELDS, filter, limitParam, after);
}
webhooks.getData().forEach(t -> dao.withHref(uriInfo, t));
return webhooks;
ListFilter filter = new ListFilter(Include.ALL);
return listInternal(uriInfo, securityContext, "", filter, limitParam, before, after);
}
@GET

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java
View File

@ -341,7 +341,7 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
@QueryParam("hardDelete")
@DefaultValue("false")
boolean hardDelete,
@Parameter(description = "Chart Id", schema = @Schema(type = "string")) @PathParam("id") String id)
@Parameter(description = "Glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id)
throws IOException {
return delete(uriInfo, securityContext, id, recursive, hardDelete, ADMIN | BOT);
}

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResource.java
View File

@ -389,7 +389,7 @@ public class GlossaryTermResource extends EntityResource<GlossaryTerm, GlossaryT
@QueryParam("hardDelete")
@DefaultValue("false")
boolean hardDelete,
@Parameter(description = "Chart Id", schema = @Schema(type = "string")) @PathParam("id") String id)
@Parameter(description = "Glossary Term Id", schema = @Schema(type = "string")) @PathParam("id") String id)
throws IOException {
return delete(uriInfo, securityContext, id, recursive, hardDelete, ADMIN | BOT);
}

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/pipelines/PipelineResource.java
View File

@ -408,7 +408,7 @@ public class PipelineResource extends EntityResource<Pipeline, PipelineRepositor
@QueryParam("hardDelete")
@DefaultValue("false")
boolean hardDelete,
@Parameter(description = "Chart Id", schema = @Schema(type = "string")) @PathParam("id") String id)
@Parameter(description = "Pipeline Id", schema = @Schema(type = "string")) @PathParam("id") String id)
throws IOException {
return delete(uriInfo, securityContext, id, false, hardDelete, ADMIN | BOT);
}

3
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/teams/RoleResource.java
View File

@ -106,7 +106,8 @@ public class RoleResource extends EntityResource<Role, RoleRepository> {
Role role = JsonUtils.readValue(roleJson, entityClass);
List<EntityReference> policies = role.getPolicies();
for (EntityReference policy : policies) {
EntityReference ref = Entity.getEntityReferenceByName(Entity.POLICY, policy.getName());
EntityReference ref =
Entity.getEntityReferenceByName(Entity.POLICY, policy.getName(), Include.NON_DELETED);
policy.setId(ref.getId());
}
dao.initSeedData(role);

362
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/types/TypeResource.java Normal file
View File

@ -0,0 +1,362 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.resources.types;
import static org.openmetadata.catalog.security.SecurityUtil.ADMIN;
import static org.openmetadata.catalog.security.SecurityUtil.BOT;
import static org.openmetadata.catalog.security.SecurityUtil.OWNER;
import com.google.inject.Inject;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.io.IOException;
import java.util.List;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.PATCH;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.catalog.CatalogApplicationConfig;
import org.openmetadata.catalog.api.CreateType;
import org.openmetadata.catalog.entity.Type;
import org.openmetadata.catalog.jdbi3.CollectionDAO;
import org.openmetadata.catalog.jdbi3.ListFilter;
import org.openmetadata.catalog.jdbi3.TypeRepository;
import org.openmetadata.catalog.resources.Collection;
import org.openmetadata.catalog.resources.EntityResource;
import org.openmetadata.catalog.security.Authorizer;
import org.openmetadata.catalog.type.EntityHistory;
import org.openmetadata.catalog.type.Include;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.ResultList;
@Path("/v1/metadata/types")
@Api(value = "Types collection", tags = "metadata")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "types")
@Slf4j
public class TypeResource extends EntityResource<Type, TypeRepository> {
public static final String COLLECTION_PATH = "v1/metadata/types/";
@Override
public Type addHref(UriInfo uriInfo, Type type) {
return type; // Nothing to do
}
@Inject
public TypeResource(CollectionDAO dao, Authorizer authorizer) {
super(Type.class, new TypeRepository(dao), authorizer);
}
@SuppressWarnings("unused") // Method used for reflection
public void initialize(CatalogApplicationConfig config) throws IOException {
// Find tag definitions and load tag categories from the json file, if necessary
List<String> jsonSchemas = EntityUtil.getJsonDataResources(".*json/schema/type/.*\\.json$");
long now = System.currentTimeMillis();
for (String jsonSchema : jsonSchemas) {
try {
List<Type> types = JsonUtils.getTypes(jsonSchema);
types.forEach(
type -> {
type.withId(UUID.randomUUID()).withUpdatedBy("admin").withUpdatedAt(now);
LOG.info("Loading from {} type {} with schema {}", jsonSchema, type.getName(), type.getSchema());
try {
this.dao.createOrUpdate(null, type);
} catch (IOException e) {
LOG.error("Error loading type {} from {}", type.getName(), jsonSchema, e);
e.printStackTrace();
}
});
} catch (Exception e) {
LOG.warn("Failed to initialize the types from jsonSchema file {}", jsonSchema, e);
}
}
}
public static class TypeList extends ResultList<Type> {
@SuppressWarnings("unused")
TypeList() {
// Empty constructor needed for deserialization
}
public TypeList(List<Type> data, String beforeCursor, String afterCursor, int total) {
super(data, beforeCursor, afterCursor, total);
}
}
public static final String FIELDS = "";
@GET
@Valid
@Operation(
summary = "List types",
tags = "metadata",
description =
"Get a list of types."
+ " Use cursor-based pagination to limit the number "
+ "entries in the list using `limit` and `before` or `after` query params.",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of types",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = TypeList.class)))
})
public ResultList<Type> list(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Limit the number types returned. (1 to 1000000, " + "default = 10)")
@DefaultValue("10")
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,
@Parameter(description = "Returns list of types before this cursor", schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(description = "Returns list of types after this cursor", schema = @Schema(type = "string"))
@QueryParam("after")
String after)
throws IOException {
ListFilter filter = new ListFilter(Include.ALL);
return super.listInternal(uriInfo, securityContext, "", filter, limitParam, before, after);
}
@GET
@Path("/{id}")
@Operation(
summary = "Get a type",
tags = "metadata",
description = "Get a type by `id`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The type",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Type.class))),
@ApiResponse(responseCode = "404", description = "Type for instance {id} is not found")
})
public Type get(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") String id,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException {
return getInternal(uriInfo, securityContext, id, fieldsParam, include);
}
@GET
@Path("/name/{name}")
@Operation(
summary = "Get a type by name",
tags = "metadata",
description = "Get a type by name.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The type",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Type.class))),
@ApiResponse(responseCode = "404", description = "Type for instance {id} is not found")
})
public Type getByName(
@Context UriInfo uriInfo,
@PathParam("name") String name,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException {
return getByNameInternal(uriInfo, securityContext, name, fieldsParam, include);
}
@GET
@Path("/{id}/versions")
@Operation(
summary = "List type versions",
tags = "metadata",
description = "Get a list of all the versions of a type identified by `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of type versions",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class)))
})
public EntityHistory listVersions(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "type Id", schema = @Schema(type = "string")) @PathParam("id") String id)
throws IOException {
return dao.listVersions(id);
}
@GET
@Path("/{id}/versions/{version}")
@Operation(
summary = "Get a version of the types",
tags = "metadata",
description = "Get a version of the type by given `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "types",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Type.class))),
@ApiResponse(
responseCode = "404",
description = "Type for instance {id} and version {version} is " + "not found")
})
public Type getVersion(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "type Id", schema = @Schema(type = "string")) @PathParam("id") String id,
@Parameter(
description = "type version number in the form `major`.`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version")
String version)
throws IOException {
return dao.getVersion(id, version);
}
@POST
@Operation(
summary = "Create a type",
tags = "metadata",
description = "Create a new type.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The type",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = CreateType.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response create(@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateType create)
throws IOException {
Type type = getType(securityContext, create);
return create(uriInfo, securityContext, type, ADMIN | BOT);
}
@PATCH
@Path("/{id}")
@Operation(
summary = "Update a type",
tags = "metadata",
description = "Update an existing type using JsonPatch.",
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
public Response updateDescription(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") String id,
@RequestBody(
description = "JsonPatch with array of operations",
content =
@Content(
mediaType = MediaType.APPLICATION_JSON_PATCH_JSON,
examples = {
@ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]")
}))
JsonPatch patch)
throws IOException {
return patchInternal(uriInfo, securityContext, id, patch);
}
@PUT
@Operation(
summary = "Create or update a type",
tags = "metadata",
description = "Create a new type, if it does not exist or update an existing type.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The type",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Type.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response createOrUpdate(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateType create) throws IOException {
Type type = getType(securityContext, create);
return createOrUpdate(uriInfo, securityContext, type, ADMIN | BOT | OWNER);
}
@DELETE
@Path("/{id}")
@Operation(
summary = "Delete a type",
tags = "metadata",
description = "Delete a type by `id`.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "type for instance {id} is not found")
})
public Response delete(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Type Id", schema = @Schema(type = "string")) @PathParam("id") String id)
throws IOException {
return delete(uriInfo, securityContext, id, false, true, ADMIN | BOT);
}
private Type getType(SecurityContext securityContext, CreateType create) {
return new Type()
.withId(UUID.randomUUID())
.withName(create.getName())
.withDisplayName(create.getDisplayName())
.withSchema(create.getSchema())
.withDescription(create.getDescription())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(System.currentTimeMillis());
}
}

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java
View File

@ -280,6 +280,11 @@ public final class EntityUtil {
return String.format("%s.%s", entityType, "version");
}
/** Entity attribute extension name formed by `entityType.attributeName`. Example - `table.<customAttributeName>` */
public static String getAttributeExtensionPrefix(String entityType, String attributeName) {
return String.format("%s.%s", entityType, attributeName);
}
public static Double getVersion(String extension) {
String[] s = extension.split("\\.");
String versionString = s[2] + "." + s[3];

60
catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonUtils.java
View File

@ -16,17 +16,26 @@ package org.openmetadata.catalog.util;
import static org.openmetadata.catalog.util.RestUtil.DATE_TIME_FORMAT;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.io.JsonStringEncoder;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.fasterxml.jackson.datatype.jsr353.JSR353Module;
import com.networknt.schema.JsonSchema;
import com.networknt.schema.JsonSchemaFactory;
import com.networknt.schema.SpecVersion.VersionFlag;
import java.io.IOException;
import java.io.StringReader;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import javax.json.Json;
import javax.json.JsonArray;
import javax.json.JsonArrayBuilder;
@ -36,10 +45,13 @@ import javax.json.JsonReader;
import javax.json.JsonStructure;
import javax.json.JsonValue;
import javax.ws.rs.core.MediaType;
import org.openmetadata.catalog.entity.Type;
public final class JsonUtils {
public static final String TYPE_ANNOTATION = "@om-type";
public static final MediaType DEFAULT_MEDIA_TYPE = MediaType.APPLICATION_JSON_TYPE;
private static final ObjectMapper OBJECT_MAPPER;
private static final JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(VersionFlag.V7);
static {
OBJECT_MAPPER = new ObjectMapper();
@ -191,4 +203,52 @@ public final class JsonUtils {
return reader.readValue();
}
}
public static String jsonToString(String json) {
return String.valueOf(JsonStringEncoder.getInstance().quoteAsString(json));
}
public static JsonSchema getJsonSchema(String schema) {
return schemaFactory.getSchema(schema);
}
public static boolean hasAnnotation(JsonNode jsonNode, String annotation) {
String comment = String.valueOf(jsonNode.get("$comment"));
return comment != null && comment.contains(annotation);
}
/**
* Get all the types from the `definitions` section of a JSON schema file that are annotated with "$comment" field set
* to "@om-type".
*/
public static List<Type> getTypes(String jsonSchemaFile) throws IOException {
JsonNode node =
OBJECT_MAPPER.readTree(
Objects.requireNonNull(JsonUtils.class.getClassLoader().getResourceAsStream(jsonSchemaFile)));
if (node.get("definitions") == null) {
return Collections.emptyList();
}
String fileName = Paths.get(jsonSchemaFile).getFileName().toString();
String jsonNamespace = fileName.replace(" ", "").replace(".json", "");
List<Type> types = new ArrayList<>();
Iterator<Entry<String, JsonNode>> definitions = node.get("definitions").fields();
while (definitions != null && definitions.hasNext()) {
Entry<String, JsonNode> entry = definitions.next();
JsonNode value = entry.getValue();
if (JsonUtils.hasAnnotation(value, JsonUtils.TYPE_ANNOTATION)) {
String description = String.valueOf(value.get("description"));
Type type =
new Type()
.withName(entry.getKey())
.withNameSpace(jsonNamespace)
.withDescription(description)
.withDisplayName(entry.getKey())
.withSchema(value.toPrettyString());
types.add(type);
}
}
return types;
}
}

9
catalog-rest-service/src/main/java/org/openmetadata/catalog/util/TypeUtil.java Normal file
View File

@ -0,0 +1,9 @@
package org.openmetadata.catalog.util;
public class TypeUtil {
private TypeUtil() {
// Private constructor for util class
}
public static void validateValue(Object value, String jsonSchema) {}
}

37
catalog-rest-service/src/main/resources/json/schema/api/createType.json Normal file
View File

@ -0,0 +1,37 @@
{
"$id": "https://open-metadata.org/schema/api/data/createAttribute.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "createType",
"description": "Create a Type to be used for extending entities.",
"type": "object",
"properties": {
"name": {
"description": "Unique name that identifies a Type.",
"$ref": "../type/basic.json#/definitions/entityName"
},
"nameSpace": {
"description": "Namespace or group to which this type belongs to.",
"type": "string",
"default" : "custom"
},
"displayName": {
"description": "Display Name that identifies this Type.",
"type": "string"
},
"description": {
"description": "Optional description of the type.",
"type": "string"
},
"schema": {
"description": "JSON schema encoded as string. This will be used to validate the type values.",
"$ref": "../type/basic.json#/definitions/jsonSchema"
}
},
"required": [
"name",
"nameSpace",
"description",
"schema"
],
"additionalProperties": false
}

12
catalog-rest-service/src/main/resources/json/schema/entity/data/table.json
View File

@ -683,7 +683,7 @@
"default": null
},
"profileSample": {
"description": "Percentage of data we want to execute the profiler and tests on. Represented in the range (0, 100].",
"description": "Percentage of data we want to execute the profiler and tests on. Represented in the range (0, 100).",
"type": "number",
"exclusiveMinimum": 0,
"maximum": 100,
@ -725,6 +725,16 @@
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
},
"customAttributes" : {
"description": "Custom attributes added to the entity",
"type" : "array",
"items" : {
"$ref" : "../type.json"
}
},
"displayConfig" : {
}
},
"required": ["id", "name", "columns"],

5
catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json
View File

@ -110,11 +110,6 @@
"changeDescription": {
"description": "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
}
},
"required": ["id", "name", "endpoint", "eventFilters"],

2
catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/amundsenConnection.json
View File

@ -43,7 +43,7 @@
"default": "false"
},
"encrypted": {
"description": "Enable Encyption for the Amundsen Neo4j Connection.",
"description": "Enable encryption for the Amundsen Neo4j Connection.",
"type": "boolean",
"default": "false"
},

4
catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/metadataESConnection.json
View File

@ -2,12 +2,12 @@
"$id": "https://open-metadata.org/schema/entity/services/connections/metadata/metadataESConnection.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "MetadataESConnection",
"description": "Metadata to ElasticSeach Connection Config",
"description": "Metadata to ElasticSearch Connection Config",
"type": "object",
"javaType": "org.openmetadata.catalog.services.connections.metadata.MetadataESConnection",
"definitions": {
"metadataESType": {
"description": "Metadata to Elastic Seach type",
"description": "Metadata to Elastic Search type",
"type": "string",
"enum": ["MetadataES"],
"default": "MetadataES"

2
catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json
View File

@ -25,7 +25,7 @@
"default": "http://localhost:8585/api"
},
"authProvider": {
"description": "OpenMetadata Server Authentication Provider. Make sure configure same auth providers as the one configured on OpenMetadaata server.",
"description": "OpenMetadata Server Authentication Provider. Make sure configure same auth providers as the one configured on OpenMetadata server.",
"type": "string",
"enum": [
"no-auth",

66
catalog-rest-service/src/main/resources/json/schema/entity/type.json Normal file
View File

@ -0,0 +1,66 @@
{
"$id": "https://open-metadata.org/schema/entity/type.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Type",
"description": "This schema defines a type entity used for extending an entity with custom attributes.",
"type": "object",
"javaType": "org.openmetadata.catalog.entity.Type",
"properties": {
"id": {
"description": "Unique identifier of the type instance.",
"$ref": "../type/basic.json#/definitions/uuid"
},
"name": {
"description": "Unique name that identifies the type.",
"$ref": "../type/basic.json#/definitions/entityName"
},
"nameSpace": {
"description": "Namespace or group to which this type belongs to.",
"type": "string",
"default" : "custom"
},
"fullyQualifiedName": {
"description": "Unique name that identifies a type in the form of `type` + `.` + `name of the type`.",
"$ref": "../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this type.",
"type": "string"
},
"description": {
"description": "Optional description of entity.",
"type": "string"
},
"schema": {
"description": "JSON schema encoded as string that defines the type. This will be used to validate the type values.",
"$ref": "../type/basic.json#/definitions/jsonSchema"
},
"version": {
"description": "Metadata version of the entity.",
"$ref": "../type/entityHistory.json#/definitions/entityVersion"
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../type/basic.json#/definitions/timestamp"
},
"updatedBy": {
"description": "User who made the update.",
"type": "string"
},
"href": {
"description": "Link to this table resource.",
"$ref": "../type/basic.json#/definitions/href"
},
"changeDescription": {
"description": "Change that lead to this version of the entity.",
"$ref": "../type/entityHistory.json#/definitions/changeDescription"
}
},
"required": [
"name",
"nameSpace",
"description",
"type"
],
"additionalProperties": false
}

31
catalog-rest-service/src/main/resources/json/schema/type/basic.json
View File

@ -4,12 +4,28 @@
"title": "Basic",
"description": "This schema defines basic common types that are used by other schemas.",
"definitions": {
"integer" : {
"$comment" : "@om-type",
"description": "An integer type.",
"type" : "integer"
},
"number" : {
"$comment" : "@om-type",
"description": "A numeric type that includes integer or floating point numbers.",
"type" : "integer"
},
"string" : {
"$comment" : "@om-type",
"description": "A String type.",
"type" : "string"
},
"uuid": {
"description": "Unique id used to identify an entity.",
"type": "string",
"format": "uuid"
},
"email": {
"$comment" : "@om-type",
"description": "Email address of a user or other entities.",
"type": "string",
"format": "email",
@ -18,6 +34,7 @@
"maxLength": 127
},
"timestamp": {
"$comment" : "@om-type",
"description": "Timestamp in Unix epoch time milliseconds.",
"@comment": "Note that during code generation this is converted into long",
"type": "integer",
@ -29,6 +46,7 @@
"format": "uri"
},
"timeInterval": {
"$comment" : "@om-type",
"type": "object",
"description": "Time interval in unixTimeMillis.",
"javaType": "org.openmetadata.catalog.type.TimeInterval",
@ -45,15 +63,18 @@
"additionalProperties": false
},
"duration": {
"$comment" : "@om-type",
"description": "Duration in ISO 8601 format in UTC. Example - 'P23DT23H'.",
"type": "string"
},
"date": {
"$comment" : "@om-type",
"description": "Date in ISO 8601 format in UTC. Example - '2018-11-13'.",
"type": "string",
"format": "date"
},
"dateTime": {
"$comment" : "@om-type",
"description": "Date and time in ISO 8601 format. Example - '2018-11-13T20:20:39+00:00'.",
"type": "string",
"format": "date-time"
@ -76,8 +97,18 @@
"maxLength": 256
},
"sqlQuery": {
"$comment" : "@om-type",
"description": "SQL query statement. Example - 'select * from orders'.",
"type": "string"
},
"markdown": {
"$comment" : "@om-type",
"description": "Text in Markdown format",
"type": "string"
},
"jsonSchema": {
"description": "JSON schema encoded as string. This will be used to validate the JSON fields using this schema.",
"type": "string"
}
}
}

15
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java
View File

@ -86,6 +86,7 @@ import org.openmetadata.catalog.CatalogApplicationTest;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.TermReference;
import org.openmetadata.catalog.api.teams.CreateTeam;
import org.openmetadata.catalog.entity.Type;
import org.openmetadata.catalog.entity.data.Database;
import org.openmetadata.catalog.entity.data.DatabaseSchema;
import org.openmetadata.catalog.entity.data.Glossary;
@ -141,7 +142,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
protected boolean supportsOwner;
protected final boolean supportsTags;
protected boolean supportsPatch = true;
protected boolean supportsSoftDelete = true;
protected boolean supportsSoftDelete;
protected boolean supportsAuthorizedMetadataOperations = true;
protected boolean supportsFieldsQueryParam = true;
protected boolean supportsEmptyDescription = true;
@ -229,6 +230,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
this.supportsFollowers = allowedFields.contains(FIELD_FOLLOWERS);
this.supportsOwner = allowedFields.contains(FIELD_OWNER);
this.supportsTags = allowedFields.contains(FIELD_TAGS);
this.supportsSoftDelete = allowedFields.contains(FIELD_DELETED);
}
@BeforeAll
@ -571,9 +573,10 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
addFollower(entityInterface.getId(), user1.getId(), CREATED, TEST_AUTH_HEADERS);
}
entityInterface = validateGetWithDifferentFields(entity, false);
entity = entityInterface.getEntity();
validateGetCommonFields(entityInterface);
entityInterface = validateGetWithDifferentFields(entityInterface.getEntity(), true);
entityInterface = validateGetWithDifferentFields(entity, true);
validateGetCommonFields(entityInterface);
}
@ -718,7 +721,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
assertFalse(entityInterface.getFullyQualifiedName().contains("\""));
// Now post entity name with dots. FullyQualifiedName must have " to escape dotted name
name = String.format("%s_foo.bar", entityType);
name = format("%s_foo.bar", entityType);
request = createRequest(name, "", null, null);
entity = createEntity(request, ADMIN_AUTH_HEADERS);
entityInterface = getEntityInterface(entity);
@ -886,7 +889,8 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
StorageService.class,
DashboardService.class,
MessagingService.class,
IngestionPipeline.class);
IngestionPipeline.class,
Type.class);
if (services.contains(entity.getClass())) {
assertNotEquals(oldVersion, entityInterface.getVersion()); // Version did change
assertEquals("updatedDescription", entityInterface.getDescription()); // Description did change
@ -1124,12 +1128,13 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
@Test
void patch_deleted_attribute_disallowed_400(TestInfo test) throws HttpResponseException, JsonProcessingException {
if (!supportsPatch) {
if (!supportsPatch || !supportsSoftDelete) {
return;
}
// `deleted` attribute can't be set to true in PATCH operation & can only be done using DELETE operation
T entity = createEntity(createRequest(getEntityName(test), "", "", null), ADMIN_AUTH_HEADERS);
EntityInterface<T> entityInterface = getEntityInterface(entity);
String json = JsonUtils.pojoToJson(entity);
entityInterface.setDeleted(true);
assertResponse(

7
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java
View File

@ -57,7 +57,6 @@ import static org.openmetadata.common.utils.CommonUtil.getDateStringByOffset;
import java.io.IOException;
import java.net.URISyntaxException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@ -695,7 +694,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
}
@Test
void put_tableJoins_200(TestInfo test) throws IOException, ParseException {
void put_tableJoins_200(TestInfo test) throws IOException {
Table table1 = createAndCheckEntity(createRequest(test, 1), ADMIN_AUTH_HEADERS);
Table table2 = createAndCheckEntity(createRequest(test, 2), ADMIN_AUTH_HEADERS);
Table table3 = createAndCheckEntity(createRequest(test, 3), ADMIN_AUTH_HEADERS);
@ -817,7 +816,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
}
@Test
void put_tableJoinsInvalidColumnName_4xx(TestInfo test) throws IOException, ParseException {
void put_tableJoinsInvalidColumnName_4xx(TestInfo test) throws IOException {
Table table1 = createAndCheckEntity(createRequest(test, 1), ADMIN_AUTH_HEADERS);
Table table2 = createAndCheckEntity(createRequest(test, 2), ADMIN_AUTH_HEADERS);
@ -855,7 +854,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
"Date range can only include past 30 days starting today");
}
public void assertColumnJoins(List<ColumnJoin> expected, TableJoins actual) throws ParseException {
public void assertColumnJoins(List<ColumnJoin> expected, TableJoins actual) {
// Table reports last 30 days of aggregated join count
assertEquals(actual.getStartDate(), getDateStringByOffset(DATE_FORMAT, RestUtil.today(0), -30));
assertEquals(30, actual.getDayCount());

1
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java
View File

@ -69,7 +69,6 @@ public class WebhookResourceTest extends EntityResourceTest<Webhook, CreateWebho
supportsAuthorizedMetadataOperations = false;
supportsPatch = false;
supportsFieldsQueryParam = false;
supportsSoftDelete = false;
}
@Test

109
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/metadata/TypeResourceTest.java Normal file
View File

@ -0,0 +1,109 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.resources.metadata;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.TestMethodOrder;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.CreateType;
import org.openmetadata.catalog.entity.Type;
import org.openmetadata.catalog.jdbi3.TypeRepository.TypeEntityInterface;
import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.resources.types.TypeResource;
import org.openmetadata.catalog.resources.types.TypeResource.TypeList;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.TestUtils;
@Slf4j
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class TypeResourceTest extends EntityResourceTest<Type, CreateType> {
public static Type INT_TYPE;
public TypeResourceTest() {
super(Entity.TYPE, Type.class, TypeList.class, "metadata/types", TypeResource.FIELDS);
supportsEmptyDescription = false;
supportsFieldsQueryParam = false;
}
@BeforeAll
public void setup(TestInfo test) throws IOException, URISyntaxException {
super.setup(test);
INT_TYPE = getEntityByName("type.basic.integer", "", ADMIN_AUTH_HEADERS);
}
@Override
public EntityInterface<Type> validateGetWithDifferentFields(Type type, boolean byName) throws HttpResponseException {
type =
byName
? getEntityByName(type.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS)
: getEntity(type.getId(), null, ADMIN_AUTH_HEADERS);
return getEntityInterface(type);
}
@Override
public CreateType createRequest(String name, String description, String displayName, EntityReference owner) {
return new CreateType()
.withName(name)
.withDescription(description)
.withDisplayName(displayName)
.withSchema(INT_TYPE.getSchema());
}
@Override
public void validateCreatedEntity(Type createdEntity, CreateType createRequest, Map<String, String> authHeaders)
throws HttpResponseException {
validateCommonEntityFields(
getEntityInterface(createdEntity), createRequest.getDescription(), TestUtils.getPrincipal(authHeaders), null);
// Entity specific validation
assertEquals(createRequest.getSchema(), createdEntity.getSchema());
// TODO
}
@Override
public void compareEntities(Type expected, Type patched, Map<String, String> authHeaders)
throws HttpResponseException {
validateCommonEntityFields(
getEntityInterface(patched), expected.getDescription(), TestUtils.getPrincipal(authHeaders), null);
// Entity specific validation
assertEquals(expected.getSchema(), patched.getSchema());
// TODO more checks
}
@Override
public EntityInterface<Type> getEntityInterface(Type entity) {
return new TypeEntityInterface(entity);
}
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
if (expected == actual) {
return;
}
assertCommonFieldChange(fieldName, expected, actual);
}
}

2
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java
View File

@ -226,7 +226,7 @@ public class TeamResourceTest extends EntityResourceTest<Team, CreateTeam> {
change);
}
private User createTeamManager(TestInfo testInfo) throws HttpResponseException, JsonProcessingException {
private User createTeamManager(TestInfo testInfo) throws HttpResponseException {
// Create a rule that can update team
Rule rule =
new Rule().withName("TeamManagerPolicy-UpdateTeam").withAllow(true).withOperation(MetadataOperation.UpdateTeam);

41
catalog-rest-service/src/test/java/org/openmetadata/catalog/util/TypeUtilTest.java Normal file
View File

@ -0,0 +1,41 @@
package org.openmetadata.catalog.util;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.networknt.schema.JsonSchema;
import com.networknt.schema.ValidationMessage;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Set;
import org.junit.jupiter.api.Test;
public class TypeUtilTest {
private static final String customAttributes;
private static final ObjectMapper mapper = new ObjectMapper();
static {
ObjectNode node = mapper.createObjectNode();
node.put("intValue", 1);
node.put("stringValue", "abc");
node.put("stringValue", "abc");
customAttributes = node.toString();
}
@Test
public void testTypeValue() throws IOException {
JsonSchema intSchema = JsonUtils.getJsonSchema("{ \"type\" : \"integer\", \"minimum\": 10}");
JsonSchema stringSchema = JsonUtils.getJsonSchema("{ \"type\" : \"string\"}");
JsonNode json = mapper.readTree(customAttributes);
Iterator<Entry<String, JsonNode>> x = json.fields();
while (x.hasNext()) {
var entry = x.next();
if (entry.getKey().equals("intValue")) {
Set<ValidationMessage> result = intSchema.validate(entry.getValue());
} else if (entry.getKey().equals("stringValue")) {
Set<ValidationMessage> result = stringSchema.validate(entry.getValue());
}
}
}
}

2
ingestion-core/src/metadata/_version.py
View File

@ -7,5 +7,5 @@ Provides metadata version information.
from incremental import Version
__version__ = Version("metadata", 0, 11, 0, dev=3)
__version__ = Version("metadata", 0, 11, 0, dev=4)
__all__ = ["__version__"]