diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v002__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v002__create_db_connection_info.sql new file mode 100644 index 00000000000..3f12882de91 --- /dev/null +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v002__create_db_connection_info.sql @@ -0,0 +1,19 @@ +-- +-- Table to be used for generic entities that are limited in number. Examples of generic entities are +-- Attribute entity, domain entities etc. +-- +-- This reduces need for defining a table per entity. +-- +CREATE TABLE IF NOT EXISTS generic_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + -- Fully qualified name formed by entityType + "." + entityName + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + PRIMARY KEY (id), + UNIQUE (fullyQualifiedName) +); + +ALTER TABLE webhook_entity +DROP COLUMN deleted; diff --git a/bootstrap/sql/org.postgresql.Driver/v002__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v002__create_db_connection_info.sql new file mode 100644 index 00000000000..d17120719b6 --- /dev/null +++ b/bootstrap/sql/org.postgresql.Driver/v002__create_db_connection_info.sql @@ -0,0 +1,19 @@ +-- +-- Table to be used for generic entities that are limited in number. Examples of generic entities are +-- Attribute entity, domain entities etc. +-- +-- This reduces need for defining a table per entity. +-- +CREATE TABLE IF NOT EXISTS generic_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + -- Fully qualified name formed by entityType + "." + entityName + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + PRIMARY KEY (id), + UNIQUE (fullyQualifiedName) +); + +ALTER TABLE webhook_entity +DROP COLUMN deleted; diff --git a/catalog-rest-service/pom.xml b/catalog-rest-service/pom.xml index 950cc16d6a5..cdceab9eab9 100644 --- a/catalog-rest-service/pom.xml +++ b/catalog-rest-service/pom.xml @@ -306,6 +306,12 @@ 20220320 + + com.networknt + json-schema-validator + 1.0.69 + + io.jsonwebtoken diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java index 08a25f73820..ec2a2bec7c1 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java @@ -64,7 +64,7 @@ public final class Entity { public static final String FIELD_DISPLAY_NAME = "displayName"; // - // Services + // Service entities // public static final String DATABASE_SERVICE = "databaseService"; public static final String MESSAGING_SERVICE = "messagingService"; @@ -73,7 +73,7 @@ public final class Entity { public static final String STORAGE_SERVICE = "storageService"; // - // Data assets + // Data asset entities // public static final String TABLE = "table"; public static final String DATABASE = "database"; @@ -94,21 +94,22 @@ public final class Entity { public static final String GLOSSARY_TERM = "glossaryTerm"; public static final String TAG = "tag"; public static final String TAG_CATEGORY = "tagCategory"; + public static final String TYPE = "type"; // - // Policies + // Policy entity // public static final String POLICY = "policy"; // - // Role, team and user + // Role, team and user entities // public static final String ROLE = "role"; public static final String USER = "user"; public static final String TEAM = "team"; // - // Operations + // Operation related entities // public static final String INGESTION_PIPELINE = "ingestionPipeline"; public static final String WEBHOOK = "webhook"; @@ -145,8 +146,15 @@ public final class Entity { entityRepository.getClass().getSimpleName()); } + public static void validateEntity(String entityType) { + String canonicalEntity = CANONICAL_ENTITY_NAME_MAP.get(entityType.toLowerCase()); + if (canonicalEntity == null) { + throw new IllegalArgumentException(CatalogExceptionMessage.invalidEntity(entityType)); + } + } + public static EntityReference getEntityReference(EntityReference ref) throws IOException { - return ref == null ? null : getEntityReferenceById(ref.getType(), ref.getId()); + return ref == null ? null : getEntityReferenceById(ref.getType(), ref.getId(), Include.NON_DELETED); } public static EntityReference getEntityReference(T entity) { @@ -154,23 +162,14 @@ public final class Entity { return getEntityRepository(entityType).getEntityInterface(entity).getEntityReference(); } - public static EntityReference getEntityReferenceById(@NonNull String entityType, @NonNull UUID id) - throws IOException { - return getEntityReferenceById(entityType, id, Include.NON_DELETED); - } - public static EntityReference getEntityReferenceById(@NonNull String entityType, @NonNull UUID id, Include include) throws IOException { - EntityDAO dao = DAO_MAP.get(entityType); - if (dao == null) { + EntityRepository repository = ENTITY_REPOSITORY_MAP.get(entityType); + if (repository == null) { throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(entityType)); } - return dao.findEntityReferenceById(id, include); - } - - public static EntityReference getEntityReferenceByName(@NonNull String entityType, @NonNull String fqn) - throws IOException { - return getEntityReferenceByName(entityType, fqn, Include.NON_DELETED); + include = repository.supportsSoftDelete ? Include.ALL : include; + return repository.dao.findEntityReferenceById(id, include); } public static EntityReference getEntityReferenceByName( diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogExceptionMessage.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogExceptionMessage.java index 88e6dac80a7..f971f9c0652 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogExceptionMessage.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogExceptionMessage.java @@ -119,4 +119,8 @@ public final class CatalogExceptionMessage { public static String entityIsNotEmpty(String entityType) { return String.format("%s is not empty", entityType); } + + public static String invalidEntity(String entity) { + return String.format("Invalid entity %s", entity); + } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java index 1c602cd627d..16966fc2789 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java @@ -33,6 +33,7 @@ import org.jdbi.v3.sqlobject.statement.SqlQuery; import org.jdbi.v3.sqlobject.statement.SqlUpdate; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.entity.Bot; +import org.openmetadata.catalog.entity.Type; import org.openmetadata.catalog.entity.data.Chart; import org.openmetadata.catalog.entity.data.Dashboard; import org.openmetadata.catalog.entity.data.Database; @@ -81,6 +82,7 @@ import org.openmetadata.catalog.jdbi3.StorageServiceRepository.StorageServiceEnt import org.openmetadata.catalog.jdbi3.TableRepository.TableEntityInterface; import org.openmetadata.catalog.jdbi3.TeamRepository.TeamEntityInterface; import org.openmetadata.catalog.jdbi3.TopicRepository.TopicEntityInterface; +import org.openmetadata.catalog.jdbi3.TypeRepository.TypeEntityInterface; import org.openmetadata.catalog.jdbi3.UserRepository.UserEntityInterface; import org.openmetadata.catalog.jdbi3.WebhookRepository.WebhookEntityInterface; import org.openmetadata.catalog.jdbi3.locator.ConnectionAwareSqlQuery; @@ -199,6 +201,9 @@ public interface CollectionDAO { @CreateSqlObject WebhookDAO webhookDAO(); + @CreateSqlObject + GenericEntityDAO genericEntityDAO(); + interface DashboardDAO extends EntityDAO { @Override default String getTableName() { @@ -1248,6 +1253,11 @@ public interface CollectionDAO { default EntityReference getEntityReference(Webhook entity) { return new WebhookEntityInterface(entity).getEntityReference(); } + + @Override + default boolean supportsSoftDelete() { + return false; + } } interface TagCategoryDAO extends EntityDAO { @@ -1710,4 +1720,31 @@ public interface CollectionDAO { + "ORDER BY eventTime ASC") List listWithoutEntityFilter(@Bind("eventType") String eventType, @Bind("timestamp") long timestamp); } + + interface GenericEntityDAO extends EntityDAO { + @Override + default String getTableName() { + return "generic_entity"; + } + + @Override + default Class getEntityClass() { + return Type.class; + } + + @Override + default String getNameColumn() { + return "fullyQualifiedName"; + } + + @Override + default EntityReference getEntityReference(Type entity) { + return new TypeEntityInterface(entity).getEntityReference(); + } + + @Override + default boolean supportsSoftDelete() { + return false; + } + } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityDAO.java index 9dea388f144..9f356e7968e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityDAO.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityDAO.java @@ -45,6 +45,10 @@ public interface EntityDAO { EntityReference getEntityReference(T entity); + default boolean supportsSoftDelete() { + return true; + } + /** Common queries for all entities implemented here. Do not override. */ @ConnectionAwareSqlUpdate(value = "INSERT INTO (json) VALUES (:json)", connectionType = MYSQL) @ConnectionAwareSqlUpdate(value = "INSERT INTO
(json) VALUES (:json :: jsonb)", connectionType = POSTGRES) @@ -113,6 +117,10 @@ public interface EntityDAO { } default String getCondition(Include include) { + if (!supportsSoftDelete()) { + return ""; + } + if (include == null || include == Include.NON_DELETED) { return "AND deleted = FALSE"; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java index 7f7a944d86c..6e805321b51 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java @@ -115,10 +115,10 @@ public abstract class EntityRepository { private final String collectionPath; private final Class entityClass; private final String entityType; - protected final EntityDAO dao; + public final EntityDAO dao; protected final CollectionDAO daoCollection; protected final List allowedFields; - protected boolean supportsSoftDelete = true; + public final boolean supportsSoftDelete; protected final boolean supportsTags; protected final boolean supportsOwner; protected final boolean supportsFollower; @@ -149,6 +149,7 @@ public abstract class EntityRepository { this.supportsTags = allowedFields.contains(FIELD_TAGS); this.supportsOwner = allowedFields.contains(FIELD_OWNER); + this.supportsSoftDelete = allowedFields.contains(FIELD_DELETED); this.supportsFollower = allowedFields.contains(FIELD_FOLLOWERS); Entity.registerEntity(entityClass, entityType, dao, this); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LineageRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LineageRepository.java index d014cde7d61..84acbaea132 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LineageRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LineageRepository.java @@ -24,6 +24,7 @@ import org.openmetadata.catalog.api.lineage.AddLineage; import org.openmetadata.catalog.type.Edge; import org.openmetadata.catalog.type.EntityLineage; import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.type.Include; import org.openmetadata.catalog.type.Relationship; public class LineageRepository { @@ -35,14 +36,14 @@ public class LineageRepository { @Transaction public EntityLineage get(String entityType, String id, int upstreamDepth, int downstreamDepth) throws IOException { - EntityReference ref = Entity.getEntityReferenceById(entityType, UUID.fromString(id)); + EntityReference ref = Entity.getEntityReferenceById(entityType, UUID.fromString(id), Include.NON_DELETED); return getLineage(ref, upstreamDepth, downstreamDepth); } @Transaction public EntityLineage getByName(String entityType, String fqn, int upstreamDepth, int downstreamDepth) throws IOException { - EntityReference ref = Entity.getEntityReferenceByName(entityType, fqn); + EntityReference ref = Entity.getEntityReferenceByName(entityType, fqn, Include.NON_DELETED); return getLineage(ref, upstreamDepth, downstreamDepth); } @@ -50,11 +51,11 @@ public class LineageRepository { public void addLineage(AddLineage addLineage) throws IOException { // Validate from entity EntityReference from = addLineage.getEdge().getFromEntity(); - from = Entity.getEntityReferenceById(from.getType(), from.getId()); + from = Entity.getEntityReferenceById(from.getType(), from.getId(), Include.NON_DELETED); // Validate to entity EntityReference to = addLineage.getEdge().getToEntity(); - to = Entity.getEntityReferenceById(to.getType(), to.getId()); + to = Entity.getEntityReferenceById(to.getType(), to.getId(), Include.NON_DELETED); // Finally, add lineage relationship dao.relationshipDAO() @@ -64,10 +65,10 @@ public class LineageRepository { @Transaction public boolean deleteLineage(String fromEntity, String fromId, String toEntity, String toId) throws IOException { // Validate from entity - EntityReference from = Entity.getEntityReferenceById(fromEntity, UUID.fromString(fromId)); + EntityReference from = Entity.getEntityReferenceById(fromEntity, UUID.fromString(fromId), Include.NON_DELETED); // Validate to entity - EntityReference to = Entity.getEntityReferenceById(toEntity, UUID.fromString(toId)); + EntityReference to = Entity.getEntityReferenceById(toEntity, UUID.fromString(toId), Include.NON_DELETED); // Finally, delete lineage relationship return dao.relationshipDAO() @@ -97,7 +98,7 @@ public class LineageRepository { // Add entityReference details for (int i = 0; i < lineage.getNodes().size(); i++) { EntityReference ref = lineage.getNodes().get(i); - ref = Entity.getEntityReferenceById(ref.getType(), ref.getId()); + ref = Entity.getEntityReferenceById(ref.getType(), ref.getId(), Include.NON_DELETED); lineage.getNodes().set(i, ref); } return lineage; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java index 81faee9a234..ad3532ebe53 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java @@ -35,6 +35,7 @@ import org.openmetadata.catalog.entity.data.MlModel; import org.openmetadata.catalog.resources.mlmodels.MlModelResource; import org.openmetadata.catalog.type.ChangeDescription; import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.type.Include; import org.openmetadata.catalog.type.MlFeature; import org.openmetadata.catalog.type.MlFeatureSource; import org.openmetadata.catalog.type.MlHyperParameter; @@ -125,7 +126,8 @@ public class MlModelRepository extends EntityRepository { private void validateMlDataSource(MlFeatureSource source) throws IOException { if (source.getDataSource() != null) { - Entity.getEntityReferenceById(source.getDataSource().getType(), source.getDataSource().getId()); + Entity.getEntityReferenceById( + source.getDataSource().getType(), source.getDataSource().getId(), Include.NON_DELETED); } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TypeRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TypeRepository.java new file mode 100644 index 00000000000..2bc3b595165 --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TypeRepository.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.jdbi3; + +import java.io.IOException; +import java.net.URI; +import java.util.UUID; +import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.entity.Type; +import org.openmetadata.catalog.resources.types.TypeResource; +import org.openmetadata.catalog.type.ChangeDescription; +import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.util.EntityInterface; +import org.openmetadata.catalog.util.EntityUtil.Fields; +import org.openmetadata.catalog.util.FullyQualifiedName; + +public class TypeRepository extends EntityRepository { + // TODO fix this + private static final String UPDATE_FIELDS = ""; + private static final String PATCH_FIELDS = ""; + + public TypeRepository(CollectionDAO dao) { + super( + TypeResource.COLLECTION_PATH, + Entity.TYPE, + Type.class, + dao.genericEntityDAO(), + dao, + PATCH_FIELDS, + UPDATE_FIELDS); + allowEdits = true; + } + + @Override + public Type setFields(Type attribute, Fields fields) throws IOException { + return attribute; + } + + @Override + public void prepare(Type type) throws IOException { + type.setFullyQualifiedName(getEntityInterface(type).getFullyQualifiedName()); + } + + @Override + public void storeEntity(Type type, boolean update) throws IOException { + URI href = type.getHref(); + type.withHref(null); + store(type.getId(), type, update); + type.withHref(href); + } + + @Override + public void storeRelationships(Type type) { + // Nothing to do + } + + @Override + public EntityInterface getEntityInterface(Type entity) { + return new TypeEntityInterface(entity); + } + + @Override + public EntityUpdater getUpdater(Type original, Type updated, Operation operation) { + return new AttributeUpdater(original, updated, operation); + } + + public static class TypeEntityInterface extends EntityInterface { + public TypeEntityInterface(Type entity) { + super(Entity.TYPE, entity); + } + + @Override + public UUID getId() { + return entity.getId(); + } + + @Override + public String getDescription() { + return entity.getDescription(); + } + + @Override + public String getDisplayName() { + return entity.getDisplayName(); + } + + @Override + public String getName() { + return entity.getName(); + } + + @Override + public Boolean isDeleted() { + return false; + } + + @Override + public EntityReference getOwner() { + return null; + } + + @Override + public String getFullyQualifiedName() { + return FullyQualifiedName.build(entityType, entity.getNameSpace(), entity.getName()); + } + + @Override + public Double getVersion() { + return entity.getVersion(); + } + + @Override + public String getUpdatedBy() { + return entity.getUpdatedBy(); + } + + @Override + public long getUpdatedAt() { + return entity.getUpdatedAt(); + } + + @Override + public URI getHref() { + return entity.getHref(); + } + + @Override + public ChangeDescription getChangeDescription() { + return entity.getChangeDescription(); + } + + @Override + public Type getEntity() { + return entity; + } + + @Override + public EntityReference getContainer() { + return null; + } + + @Override + public void setId(UUID id) { + entity.setId(id); + } + + @Override + public void setDescription(String description) { + entity.setDescription(description); + } + + @Override + public void setDisplayName(String displayName) { + entity.setDisplayName(displayName); + } + + @Override + public void setName(String name) { + entity.setName(name); + } + + @Override + public void setUpdateDetails(String updatedBy, long updatedAt) { + entity.setUpdatedBy(updatedBy); + entity.setUpdatedAt(updatedAt); + } + + @Override + public void setChangeDescription(Double newVersion, ChangeDescription changeDescription) { + entity.setVersion(newVersion); + entity.setChangeDescription(changeDescription); + } + + @Override + public void setDeleted(boolean flag) {} + + @Override + public Type withHref(URI href) { + return entity.withHref(href); + } + } + + /** Handles entity updated from PUT and POST operation. */ + public class AttributeUpdater extends EntityUpdater { + public AttributeUpdater(Type original, Type updated, Operation operation) { + super(original, updated, operation); + } + } +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UsageRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UsageRepository.java index ebc350b9999..431c5caaf9f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UsageRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UsageRepository.java @@ -42,14 +42,14 @@ public class UsageRepository { @Transaction public EntityUsage get(String entityType, String id, String date, int days) throws IOException { - EntityReference ref = Entity.getEntityReferenceById(entityType, UUID.fromString(id)); + EntityReference ref = Entity.getEntityReferenceById(entityType, UUID.fromString(id), Include.NON_DELETED); List usageDetails = dao.usageDAO().getUsageById(id, date, days - 1); return new EntityUsage().withUsage(usageDetails).withEntity(ref); } @Transaction public EntityUsage getByName(String entityType, String fqn, String date, int days) throws IOException { - EntityReference ref = Entity.getEntityReferenceByName(entityType, fqn); + EntityReference ref = Entity.getEntityReferenceByName(entityType, fqn, Include.NON_DELETED); List usageDetails = dao.usageDAO().getUsageById(ref.getId().toString(), date, days - 1); return new EntityUsage().withUsage(usageDetails).withEntity(ref); } @@ -57,13 +57,13 @@ public class UsageRepository { @Transaction public void create(String entityType, String id, DailyCount usage) throws IOException { // Validate data entity for which usage is being collected - Entity.getEntityReferenceById(entityType, UUID.fromString(id)); + Entity.getEntityReferenceById(entityType, UUID.fromString(id), Include.NON_DELETED); addUsage(entityType, id, usage); } @Transaction public void createByName(String entityType, String fullyQualifiedName, DailyCount usage) throws IOException { - EntityReference ref = Entity.getEntityReferenceByName(entityType, fullyQualifiedName); + EntityReference ref = Entity.getEntityReferenceByName(entityType, fullyQualifiedName, Include.NON_DELETED); addUsage(entityType, ref.getId().toString(), usage); LOG.info("Usage successfully posted by name"); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/WebhookRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/WebhookRepository.java index 264f17c6203..8e8f0803a56 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/WebhookRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/WebhookRepository.java @@ -184,7 +184,7 @@ public class WebhookRepository extends EntityRepository { @Override public Boolean isDeleted() { - return entity.getDeleted(); + return false; } @Override @@ -256,7 +256,7 @@ public class WebhookRepository extends EntityRepository { @Override public void setDeleted(boolean flag) { - entity.setDeleted(flag); + /* soft-delete not supported */ } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java index 45824493f30..db5ed4053b5 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/WebhookResource.java @@ -56,8 +56,6 @@ import org.openmetadata.catalog.type.Include; import org.openmetadata.catalog.type.Webhook; import org.openmetadata.catalog.type.Webhook.Status; import org.openmetadata.catalog.util.EntityUtil; -import org.openmetadata.catalog.util.EntityUtil.Fields; -import org.openmetadata.catalog.util.RestUtil; import org.openmetadata.catalog.util.ResultList; @Path("/v1/webhook") @@ -120,16 +118,8 @@ public class WebhookResource extends EntityResource @DefaultValue("non-deleted") Include include) throws IOException { - RestUtil.validateCursors(before, after); - ListFilter filter = new ListFilter(include); - ResultList webhooks; - if (before != null) { // Reverse paging - webhooks = dao.listBefore(uriInfo, Fields.EMPTY_FIELDS, filter, limitParam, before); - } else { // Forward paging or first page - webhooks = dao.listAfter(uriInfo, Fields.EMPTY_FIELDS, filter, limitParam, after); - } - webhooks.getData().forEach(t -> dao.withHref(uriInfo, t)); - return webhooks; + ListFilter filter = new ListFilter(Include.ALL); + return listInternal(uriInfo, securityContext, "", filter, limitParam, before, after); } @GET diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java index 1d62ccf92c8..225f6ecebc9 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java @@ -341,7 +341,7 @@ public class GlossaryResource extends EntityResource { Role role = JsonUtils.readValue(roleJson, entityClass); List policies = role.getPolicies(); for (EntityReference policy : policies) { - EntityReference ref = Entity.getEntityReferenceByName(Entity.POLICY, policy.getName()); + EntityReference ref = + Entity.getEntityReferenceByName(Entity.POLICY, policy.getName(), Include.NON_DELETED); policy.setId(ref.getId()); } dao.initSeedData(role); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/types/TypeResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/types/TypeResource.java new file mode 100644 index 00000000000..884163d7a71 --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/types/TypeResource.java @@ -0,0 +1,362 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.types; + +import static org.openmetadata.catalog.security.SecurityUtil.ADMIN; +import static org.openmetadata.catalog.security.SecurityUtil.BOT; +import static org.openmetadata.catalog.security.SecurityUtil.OWNER; + +import com.google.inject.Inject; +import io.swagger.annotations.Api; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import java.io.IOException; +import java.util.List; +import java.util.UUID; +import javax.json.JsonPatch; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.PATCH; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.catalog.CatalogApplicationConfig; +import org.openmetadata.catalog.api.CreateType; +import org.openmetadata.catalog.entity.Type; +import org.openmetadata.catalog.jdbi3.CollectionDAO; +import org.openmetadata.catalog.jdbi3.ListFilter; +import org.openmetadata.catalog.jdbi3.TypeRepository; +import org.openmetadata.catalog.resources.Collection; +import org.openmetadata.catalog.resources.EntityResource; +import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.type.EntityHistory; +import org.openmetadata.catalog.type.Include; +import org.openmetadata.catalog.util.EntityUtil; +import org.openmetadata.catalog.util.JsonUtils; +import org.openmetadata.catalog.util.ResultList; + +@Path("/v1/metadata/types") +@Api(value = "Types collection", tags = "metadata") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "types") +@Slf4j +public class TypeResource extends EntityResource { + public static final String COLLECTION_PATH = "v1/metadata/types/"; + + @Override + public Type addHref(UriInfo uriInfo, Type type) { + return type; // Nothing to do + } + + @Inject + public TypeResource(CollectionDAO dao, Authorizer authorizer) { + super(Type.class, new TypeRepository(dao), authorizer); + } + + @SuppressWarnings("unused") // Method used for reflection + public void initialize(CatalogApplicationConfig config) throws IOException { + // Find tag definitions and load tag categories from the json file, if necessary + List jsonSchemas = EntityUtil.getJsonDataResources(".*json/schema/type/.*\\.json$"); + long now = System.currentTimeMillis(); + for (String jsonSchema : jsonSchemas) { + try { + List types = JsonUtils.getTypes(jsonSchema); + types.forEach( + type -> { + type.withId(UUID.randomUUID()).withUpdatedBy("admin").withUpdatedAt(now); + LOG.info("Loading from {} type {} with schema {}", jsonSchema, type.getName(), type.getSchema()); + try { + this.dao.createOrUpdate(null, type); + } catch (IOException e) { + LOG.error("Error loading type {} from {}", type.getName(), jsonSchema, e); + e.printStackTrace(); + } + }); + } catch (Exception e) { + LOG.warn("Failed to initialize the types from jsonSchema file {}", jsonSchema, e); + } + } + } + + public static class TypeList extends ResultList { + @SuppressWarnings("unused") + TypeList() { + // Empty constructor needed for deserialization + } + + public TypeList(List data, String beforeCursor, String afterCursor, int total) { + super(data, beforeCursor, afterCursor, total); + } + } + + public static final String FIELDS = ""; + + @GET + @Valid + @Operation( + summary = "List types", + tags = "metadata", + description = + "Get a list of types." + + " Use cursor-based pagination to limit the number " + + "entries in the list using `limit` and `before` or `after` query params.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of types", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = TypeList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Limit the number types returned. (1 to 1000000, " + "default = 10)") + @DefaultValue("10") + @Min(0) + @Max(1000000) + @QueryParam("limit") + int limitParam, + @Parameter(description = "Returns list of types before this cursor", schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter(description = "Returns list of types after this cursor", schema = @Schema(type = "string")) + @QueryParam("after") + String after) + throws IOException { + ListFilter filter = new ListFilter(Include.ALL); + return super.listInternal(uriInfo, securityContext, "", filter, limitParam, before, after); + } + + @GET + @Path("/{id}") + @Operation( + summary = "Get a type", + tags = "metadata", + description = "Get a type by `id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The type", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Type.class))), + @ApiResponse(responseCode = "404", description = "Type for instance {id} is not found") + }) + public Type get( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") String id, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException { + return getInternal(uriInfo, securityContext, id, fieldsParam, include); + } + + @GET + @Path("/name/{name}") + @Operation( + summary = "Get a type by name", + tags = "metadata", + description = "Get a type by name.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The type", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Type.class))), + @ApiResponse(responseCode = "404", description = "Type for instance {id} is not found") + }) + public Type getByName( + @Context UriInfo uriInfo, + @PathParam("name") String name, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException { + return getByNameInternal(uriInfo, securityContext, name, fieldsParam, include); + } + + @GET + @Path("/{id}/versions") + @Operation( + summary = "List type versions", + tags = "metadata", + description = "Get a list of all the versions of a type identified by `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of type versions", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class))) + }) + public EntityHistory listVersions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "type Id", schema = @Schema(type = "string")) @PathParam("id") String id) + throws IOException { + return dao.listVersions(id); + } + + @GET + @Path("/{id}/versions/{version}") + @Operation( + summary = "Get a version of the types", + tags = "metadata", + description = "Get a version of the type by given `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "types", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Type.class))), + @ApiResponse( + responseCode = "404", + description = "Type for instance {id} and version {version} is " + "not found") + }) + public Type getVersion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "type Id", schema = @Schema(type = "string")) @PathParam("id") String id, + @Parameter( + description = "type version number in the form `major`.`minor`", + schema = @Schema(type = "string", example = "0.1 or 1.1")) + @PathParam("version") + String version) + throws IOException { + return dao.getVersion(id, version); + } + + @POST + @Operation( + summary = "Create a type", + tags = "metadata", + description = "Create a new type.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The type", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = CreateType.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response create(@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateType create) + throws IOException { + Type type = getType(securityContext, create); + return create(uriInfo, securityContext, type, ADMIN | BOT); + } + + @PATCH + @Path("/{id}") + @Operation( + summary = "Update a type", + tags = "metadata", + description = "Update an existing type using JsonPatch.", + externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902")) + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + public Response updateDescription( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") String id, + @RequestBody( + description = "JsonPatch with array of operations", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON_PATCH_JSON, + examples = { + @ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]") + })) + JsonPatch patch) + throws IOException { + return patchInternal(uriInfo, securityContext, id, patch); + } + + @PUT + @Operation( + summary = "Create or update a type", + tags = "metadata", + description = "Create a new type, if it does not exist or update an existing type.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The type", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Type.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response createOrUpdate( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateType create) throws IOException { + Type type = getType(securityContext, create); + return createOrUpdate(uriInfo, securityContext, type, ADMIN | BOT | OWNER); + } + + @DELETE + @Path("/{id}") + @Operation( + summary = "Delete a type", + tags = "metadata", + description = "Delete a type by `id`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "type for instance {id} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Type Id", schema = @Schema(type = "string")) @PathParam("id") String id) + throws IOException { + return delete(uriInfo, securityContext, id, false, true, ADMIN | BOT); + } + + private Type getType(SecurityContext securityContext, CreateType create) { + return new Type() + .withId(UUID.randomUUID()) + .withName(create.getName()) + .withDisplayName(create.getDisplayName()) + .withSchema(create.getSchema()) + .withDescription(create.getDescription()) + .withUpdatedBy(securityContext.getUserPrincipal().getName()) + .withUpdatedAt(System.currentTimeMillis()); + } +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java index 1e0261c39cb..23587b15243 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java @@ -280,6 +280,11 @@ public final class EntityUtil { return String.format("%s.%s", entityType, "version"); } + /** Entity attribute extension name formed by `entityType.attributeName`. Example - `table.` */ + public static String getAttributeExtensionPrefix(String entityType, String attributeName) { + return String.format("%s.%s", entityType, attributeName); + } + public static Double getVersion(String extension) { String[] s = extension.split("\\."); String versionString = s[2] + "." + s[3]; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonUtils.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonUtils.java index 3c4b686c553..6bf9484e26a 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonUtils.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonUtils.java @@ -16,17 +16,26 @@ package org.openmetadata.catalog.util; import static org.openmetadata.catalog.util.RestUtil.DATE_TIME_FORMAT; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.io.JsonStringEncoder; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.type.TypeFactory; import com.fasterxml.jackson.datatype.jsr353.JSR353Module; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion.VersionFlag; import java.io.IOException; import java.io.StringReader; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonArrayBuilder; @@ -36,10 +45,13 @@ import javax.json.JsonReader; import javax.json.JsonStructure; import javax.json.JsonValue; import javax.ws.rs.core.MediaType; +import org.openmetadata.catalog.entity.Type; public final class JsonUtils { + public static final String TYPE_ANNOTATION = "@om-type"; public static final MediaType DEFAULT_MEDIA_TYPE = MediaType.APPLICATION_JSON_TYPE; private static final ObjectMapper OBJECT_MAPPER; + private static final JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(VersionFlag.V7); static { OBJECT_MAPPER = new ObjectMapper(); @@ -191,4 +203,52 @@ public final class JsonUtils { return reader.readValue(); } } + + public static String jsonToString(String json) { + return String.valueOf(JsonStringEncoder.getInstance().quoteAsString(json)); + } + + public static JsonSchema getJsonSchema(String schema) { + return schemaFactory.getSchema(schema); + } + + public static boolean hasAnnotation(JsonNode jsonNode, String annotation) { + String comment = String.valueOf(jsonNode.get("$comment")); + return comment != null && comment.contains(annotation); + } + + /** + * Get all the types from the `definitions` section of a JSON schema file that are annotated with "$comment" field set + * to "@om-type". + */ + public static List getTypes(String jsonSchemaFile) throws IOException { + JsonNode node = + OBJECT_MAPPER.readTree( + Objects.requireNonNull(JsonUtils.class.getClassLoader().getResourceAsStream(jsonSchemaFile))); + if (node.get("definitions") == null) { + return Collections.emptyList(); + } + + String fileName = Paths.get(jsonSchemaFile).getFileName().toString(); + String jsonNamespace = fileName.replace(" ", "").replace(".json", ""); + + List types = new ArrayList<>(); + Iterator> definitions = node.get("definitions").fields(); + while (definitions != null && definitions.hasNext()) { + Entry entry = definitions.next(); + JsonNode value = entry.getValue(); + if (JsonUtils.hasAnnotation(value, JsonUtils.TYPE_ANNOTATION)) { + String description = String.valueOf(value.get("description")); + Type type = + new Type() + .withName(entry.getKey()) + .withNameSpace(jsonNamespace) + .withDescription(description) + .withDisplayName(entry.getKey()) + .withSchema(value.toPrettyString()); + types.add(type); + } + } + return types; + } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/TypeUtil.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/TypeUtil.java new file mode 100644 index 00000000000..d3482c515bc --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/TypeUtil.java @@ -0,0 +1,9 @@ +package org.openmetadata.catalog.util; + +public class TypeUtil { + private TypeUtil() { + // Private constructor for util class + } + + public static void validateValue(Object value, String jsonSchema) {} +} diff --git a/catalog-rest-service/src/main/resources/json/schema/api/createType.json b/catalog-rest-service/src/main/resources/json/schema/api/createType.json new file mode 100644 index 00000000000..b31c1fcf63c --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/api/createType.json @@ -0,0 +1,37 @@ +{ + "$id": "https://open-metadata.org/schema/api/data/createAttribute.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "createType", + "description": "Create a Type to be used for extending entities.", + "type": "object", + "properties": { + "name": { + "description": "Unique name that identifies a Type.", + "$ref": "../type/basic.json#/definitions/entityName" + }, + "nameSpace": { + "description": "Namespace or group to which this type belongs to.", + "type": "string", + "default" : "custom" + }, + "displayName": { + "description": "Display Name that identifies this Type.", + "type": "string" + }, + "description": { + "description": "Optional description of the type.", + "type": "string" + }, + "schema": { + "description": "JSON schema encoded as string. This will be used to validate the type values.", + "$ref": "../type/basic.json#/definitions/jsonSchema" + } + }, + "required": [ + "name", + "nameSpace", + "description", + "schema" + ], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json index 4227956a559..1f98dba364c 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json @@ -683,7 +683,7 @@ "default": null }, "profileSample": { - "description": "Percentage of data we want to execute the profiler and tests on. Represented in the range (0, 100].", + "description": "Percentage of data we want to execute the profiler and tests on. Represented in the range (0, 100).", "type": "number", "exclusiveMinimum": 0, "maximum": 100, @@ -725,6 +725,16 @@ "description": "When `true` indicates the entity has been soft deleted.", "type": "boolean", "default": false + }, + "customAttributes" : { + "description": "Custom attributes added to the entity", + "type" : "array", + "items" : { + "$ref" : "../type.json" + } + }, + "displayConfig" : { + } }, "required": ["id", "name", "columns"], diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json b/catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json index d7b31aa2605..8b1a2ccd1d5 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json @@ -110,11 +110,6 @@ "changeDescription": { "description": "Change that lead to this version of the entity.", "$ref": "../../type/entityHistory.json#/definitions/changeDescription" - }, - "deleted": { - "description": "When `true` indicates the entity has been soft deleted.", - "type": "boolean", - "default": false } }, "required": ["id", "name", "endpoint", "eventFilters"], diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/amundsenConnection.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/amundsenConnection.json index 9fb1eb84398..89e0c0b3dc5 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/amundsenConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/amundsenConnection.json @@ -43,7 +43,7 @@ "default": "false" }, "encrypted": { - "description": "Enable Encyption for the Amundsen Neo4j Connection.", + "description": "Enable encryption for the Amundsen Neo4j Connection.", "type": "boolean", "default": "false" }, diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/metadataESConnection.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/metadataESConnection.json index 2af1d86ef9b..387da693b82 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/metadataESConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/metadataESConnection.json @@ -2,12 +2,12 @@ "$id": "https://open-metadata.org/schema/entity/services/connections/metadata/metadataESConnection.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "MetadataESConnection", - "description": "Metadata to ElasticSeach Connection Config", + "description": "Metadata to ElasticSearch Connection Config", "type": "object", "javaType": "org.openmetadata.catalog.services.connections.metadata.MetadataESConnection", "definitions": { "metadataESType": { - "description": "Metadata to Elastic Seach type", + "description": "Metadata to Elastic Search type", "type": "string", "enum": ["MetadataES"], "default": "MetadataES" diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json index 868cff208eb..abe485efe98 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json @@ -25,7 +25,7 @@ "default": "http://localhost:8585/api" }, "authProvider": { - "description": "OpenMetadata Server Authentication Provider. Make sure configure same auth providers as the one configured on OpenMetadaata server.", + "description": "OpenMetadata Server Authentication Provider. Make sure configure same auth providers as the one configured on OpenMetadata server.", "type": "string", "enum": [ "no-auth", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/type.json b/catalog-rest-service/src/main/resources/json/schema/entity/type.json new file mode 100644 index 00000000000..9d2fa020b49 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/entity/type.json @@ -0,0 +1,66 @@ +{ + "$id": "https://open-metadata.org/schema/entity/type.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Type", + "description": "This schema defines a type entity used for extending an entity with custom attributes.", + "type": "object", + "javaType": "org.openmetadata.catalog.entity.Type", + "properties": { + "id": { + "description": "Unique identifier of the type instance.", + "$ref": "../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Unique name that identifies the type.", + "$ref": "../type/basic.json#/definitions/entityName" + }, + "nameSpace": { + "description": "Namespace or group to which this type belongs to.", + "type": "string", + "default" : "custom" + }, + "fullyQualifiedName": { + "description": "Unique name that identifies a type in the form of `type` + `.` + `name of the type`.", + "$ref": "../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display Name that identifies this type.", + "type": "string" + }, + "description": { + "description": "Optional description of entity.", + "type": "string" + }, + "schema": { + "description": "JSON schema encoded as string that defines the type. This will be used to validate the type values.", + "$ref": "../type/basic.json#/definitions/jsonSchema" + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../type/entityHistory.json#/definitions/entityVersion" + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "href": { + "description": "Link to this table resource.", + "$ref": "../type/basic.json#/definitions/href" + }, + "changeDescription": { + "description": "Change that lead to this version of the entity.", + "$ref": "../type/entityHistory.json#/definitions/changeDescription" + } + }, + "required": [ + "name", + "nameSpace", + "description", + "type" + ], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/type/basic.json b/catalog-rest-service/src/main/resources/json/schema/type/basic.json index ac0d9430981..1ea3f4f0c56 100644 --- a/catalog-rest-service/src/main/resources/json/schema/type/basic.json +++ b/catalog-rest-service/src/main/resources/json/schema/type/basic.json @@ -4,12 +4,28 @@ "title": "Basic", "description": "This schema defines basic common types that are used by other schemas.", "definitions": { + "integer" : { + "$comment" : "@om-type", + "description": "An integer type.", + "type" : "integer" + }, + "number" : { + "$comment" : "@om-type", + "description": "A numeric type that includes integer or floating point numbers.", + "type" : "integer" + }, + "string" : { + "$comment" : "@om-type", + "description": "A String type.", + "type" : "string" + }, "uuid": { "description": "Unique id used to identify an entity.", "type": "string", "format": "uuid" }, "email": { + "$comment" : "@om-type", "description": "Email address of a user or other entities.", "type": "string", "format": "email", @@ -18,6 +34,7 @@ "maxLength": 127 }, "timestamp": { + "$comment" : "@om-type", "description": "Timestamp in Unix epoch time milliseconds.", "@comment": "Note that during code generation this is converted into long", "type": "integer", @@ -29,6 +46,7 @@ "format": "uri" }, "timeInterval": { + "$comment" : "@om-type", "type": "object", "description": "Time interval in unixTimeMillis.", "javaType": "org.openmetadata.catalog.type.TimeInterval", @@ -45,15 +63,18 @@ "additionalProperties": false }, "duration": { + "$comment" : "@om-type", "description": "Duration in ISO 8601 format in UTC. Example - 'P23DT23H'.", "type": "string" }, "date": { + "$comment" : "@om-type", "description": "Date in ISO 8601 format in UTC. Example - '2018-11-13'.", "type": "string", "format": "date" }, "dateTime": { + "$comment" : "@om-type", "description": "Date and time in ISO 8601 format. Example - '2018-11-13T20:20:39+00:00'.", "type": "string", "format": "date-time" @@ -76,8 +97,18 @@ "maxLength": 256 }, "sqlQuery": { + "$comment" : "@om-type", "description": "SQL query statement. Example - 'select * from orders'.", "type": "string" + }, + "markdown": { + "$comment" : "@om-type", + "description": "Text in Markdown format", + "type": "string" + }, + "jsonSchema": { + "description": "JSON schema encoded as string. This will be used to validate the JSON fields using this schema.", + "type": "string" } } } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java index bdd1acfc032..8c2b2701ec7 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java @@ -86,6 +86,7 @@ import org.openmetadata.catalog.CatalogApplicationTest; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.api.data.TermReference; import org.openmetadata.catalog.api.teams.CreateTeam; +import org.openmetadata.catalog.entity.Type; import org.openmetadata.catalog.entity.data.Database; import org.openmetadata.catalog.entity.data.DatabaseSchema; import org.openmetadata.catalog.entity.data.Glossary; @@ -141,7 +142,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { protected boolean supportsOwner; protected final boolean supportsTags; protected boolean supportsPatch = true; - protected boolean supportsSoftDelete = true; + protected boolean supportsSoftDelete; protected boolean supportsAuthorizedMetadataOperations = true; protected boolean supportsFieldsQueryParam = true; protected boolean supportsEmptyDescription = true; @@ -229,6 +230,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { this.supportsFollowers = allowedFields.contains(FIELD_FOLLOWERS); this.supportsOwner = allowedFields.contains(FIELD_OWNER); this.supportsTags = allowedFields.contains(FIELD_TAGS); + this.supportsSoftDelete = allowedFields.contains(FIELD_DELETED); } @BeforeAll @@ -571,9 +573,10 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { addFollower(entityInterface.getId(), user1.getId(), CREATED, TEST_AUTH_HEADERS); } entityInterface = validateGetWithDifferentFields(entity, false); + entity = entityInterface.getEntity(); validateGetCommonFields(entityInterface); - entityInterface = validateGetWithDifferentFields(entityInterface.getEntity(), true); + entityInterface = validateGetWithDifferentFields(entity, true); validateGetCommonFields(entityInterface); } @@ -718,7 +721,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { assertFalse(entityInterface.getFullyQualifiedName().contains("\"")); // Now post entity name with dots. FullyQualifiedName must have " to escape dotted name - name = String.format("%s_foo.bar", entityType); + name = format("%s_foo.bar", entityType); request = createRequest(name, "", null, null); entity = createEntity(request, ADMIN_AUTH_HEADERS); entityInterface = getEntityInterface(entity); @@ -886,7 +889,8 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { StorageService.class, DashboardService.class, MessagingService.class, - IngestionPipeline.class); + IngestionPipeline.class, + Type.class); if (services.contains(entity.getClass())) { assertNotEquals(oldVersion, entityInterface.getVersion()); // Version did change assertEquals("updatedDescription", entityInterface.getDescription()); // Description did change @@ -1124,12 +1128,13 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { @Test void patch_deleted_attribute_disallowed_400(TestInfo test) throws HttpResponseException, JsonProcessingException { - if (!supportsPatch) { + if (!supportsPatch || !supportsSoftDelete) { return; } // `deleted` attribute can't be set to true in PATCH operation & can only be done using DELETE operation T entity = createEntity(createRequest(getEntityName(test), "", "", null), ADMIN_AUTH_HEADERS); EntityInterface entityInterface = getEntityInterface(entity); + String json = JsonUtils.pojoToJson(entity); entityInterface.setDeleted(true); assertResponse( diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java index 4cb477808a7..c35e46bd448 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java @@ -57,7 +57,6 @@ import static org.openmetadata.common.utils.CommonUtil.getDateStringByOffset; import java.io.IOException; import java.net.URISyntaxException; -import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -695,7 +694,7 @@ public class TableResourceTest extends EntityResourceTest { } @Test - void put_tableJoins_200(TestInfo test) throws IOException, ParseException { + void put_tableJoins_200(TestInfo test) throws IOException { Table table1 = createAndCheckEntity(createRequest(test, 1), ADMIN_AUTH_HEADERS); Table table2 = createAndCheckEntity(createRequest(test, 2), ADMIN_AUTH_HEADERS); Table table3 = createAndCheckEntity(createRequest(test, 3), ADMIN_AUTH_HEADERS); @@ -817,7 +816,7 @@ public class TableResourceTest extends EntityResourceTest { } @Test - void put_tableJoinsInvalidColumnName_4xx(TestInfo test) throws IOException, ParseException { + void put_tableJoinsInvalidColumnName_4xx(TestInfo test) throws IOException { Table table1 = createAndCheckEntity(createRequest(test, 1), ADMIN_AUTH_HEADERS); Table table2 = createAndCheckEntity(createRequest(test, 2), ADMIN_AUTH_HEADERS); @@ -855,7 +854,7 @@ public class TableResourceTest extends EntityResourceTest { "Date range can only include past 30 days starting today"); } - public void assertColumnJoins(List expected, TableJoins actual) throws ParseException { + public void assertColumnJoins(List expected, TableJoins actual) { // Table reports last 30 days of aggregated join count assertEquals(actual.getStartDate(), getDateStringByOffset(DATE_FORMAT, RestUtil.today(0), -30)); assertEquals(30, actual.getDayCount()); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java index 9552bfdf8c8..013ecde1a4b 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java @@ -69,7 +69,6 @@ public class WebhookResourceTest extends EntityResourceTest { + public static Type INT_TYPE; + + public TypeResourceTest() { + super(Entity.TYPE, Type.class, TypeList.class, "metadata/types", TypeResource.FIELDS); + supportsEmptyDescription = false; + supportsFieldsQueryParam = false; + } + + @BeforeAll + public void setup(TestInfo test) throws IOException, URISyntaxException { + super.setup(test); + INT_TYPE = getEntityByName("type.basic.integer", "", ADMIN_AUTH_HEADERS); + } + + @Override + public EntityInterface validateGetWithDifferentFields(Type type, boolean byName) throws HttpResponseException { + type = + byName + ? getEntityByName(type.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS) + : getEntity(type.getId(), null, ADMIN_AUTH_HEADERS); + + return getEntityInterface(type); + } + + @Override + public CreateType createRequest(String name, String description, String displayName, EntityReference owner) { + return new CreateType() + .withName(name) + .withDescription(description) + .withDisplayName(displayName) + .withSchema(INT_TYPE.getSchema()); + } + + @Override + public void validateCreatedEntity(Type createdEntity, CreateType createRequest, Map authHeaders) + throws HttpResponseException { + validateCommonEntityFields( + getEntityInterface(createdEntity), createRequest.getDescription(), TestUtils.getPrincipal(authHeaders), null); + + // Entity specific validation + assertEquals(createRequest.getSchema(), createdEntity.getSchema()); + // TODO + } + + @Override + public void compareEntities(Type expected, Type patched, Map authHeaders) + throws HttpResponseException { + validateCommonEntityFields( + getEntityInterface(patched), expected.getDescription(), TestUtils.getPrincipal(authHeaders), null); + + // Entity specific validation + assertEquals(expected.getSchema(), patched.getSchema()); + // TODO more checks + } + + @Override + public EntityInterface getEntityInterface(Type entity) { + return new TypeEntityInterface(entity); + } + + @Override + public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException { + if (expected == actual) { + return; + } + assertCommonFieldChange(fieldName, expected, actual); + } +} diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java index 35b065925f3..31800bd66c8 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java @@ -226,7 +226,7 @@ public class TeamResourceTest extends EntityResourceTest { change); } - private User createTeamManager(TestInfo testInfo) throws HttpResponseException, JsonProcessingException { + private User createTeamManager(TestInfo testInfo) throws HttpResponseException { // Create a rule that can update team Rule rule = new Rule().withName("TeamManagerPolicy-UpdateTeam").withAllow(true).withOperation(MetadataOperation.UpdateTeam); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/util/TypeUtilTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/util/TypeUtilTest.java new file mode 100644 index 00000000000..e3d78d4ef76 --- /dev/null +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/util/TypeUtilTest.java @@ -0,0 +1,41 @@ +package org.openmetadata.catalog.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.ValidationMessage; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Set; +import org.junit.jupiter.api.Test; + +public class TypeUtilTest { + private static final String customAttributes; + private static final ObjectMapper mapper = new ObjectMapper(); + + static { + ObjectNode node = mapper.createObjectNode(); + node.put("intValue", 1); + node.put("stringValue", "abc"); + node.put("stringValue", "abc"); + customAttributes = node.toString(); + } + + @Test + public void testTypeValue() throws IOException { + JsonSchema intSchema = JsonUtils.getJsonSchema("{ \"type\" : \"integer\", \"minimum\": 10}"); + JsonSchema stringSchema = JsonUtils.getJsonSchema("{ \"type\" : \"string\"}"); + JsonNode json = mapper.readTree(customAttributes); + Iterator> x = json.fields(); + while (x.hasNext()) { + var entry = x.next(); + if (entry.getKey().equals("intValue")) { + Set result = intSchema.validate(entry.getValue()); + } else if (entry.getKey().equals("stringValue")) { + Set result = stringSchema.validate(entry.getValue()); + } + } + } +} diff --git a/ingestion-core/src/metadata/_version.py b/ingestion-core/src/metadata/_version.py index 1b48c1567f5..ff26191c876 100644 --- a/ingestion-core/src/metadata/_version.py +++ b/ingestion-core/src/metadata/_version.py @@ -7,5 +7,5 @@ Provides metadata version information. from incremental import Version -__version__ = Version("metadata", 0, 11, 0, dev=3) +__version__ = Version("metadata", 0, 11, 0, dev=4) __all__ = ["__version__"]