From 97140e13fd3ba7ab2462b51fa470caf5a38c9c20 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 3 Jul 2023 15:42:35 -0700 Subject: [PATCH] 12142.task2 - Add domain and data product properties to entities (#12255) --- .../v014__create_db_connection_info.sql | 14 +- .../v014__create_db_connection_info.sql | 14 +- .../java/org/openmetadata/service/Entity.java | 3 + .../service/jdbi3/CollectionDAO.java | 26 ++ .../service/jdbi3/DataProductRepository.java | 126 +++++++ .../service/jdbi3/DomainRepository.java | 5 +- .../service/jdbi3/EntityRepository.java | 111 +++++- .../service/resources/EntityResource.java | 4 +- .../resources/charts/ChartResource.java | 2 +- .../dashboards/DashboardResource.java | 3 +- .../resources/databases/DatabaseResource.java | 2 +- .../databases/DatabaseSchemaResource.java | 2 +- .../resources/databases/TableResource.java | 7 +- .../DashboardDataModelResource.java | 2 +- .../domains/DataProductResource.java | 341 ++++++++++++++++++ .../resources/domains/DomainResource.java | 4 +- .../resources/dqtests/TestCaseResource.java | 1 - .../resources/glossary/GlossaryResource.java | 2 +- .../glossary/GlossaryTermResource.java | 2 +- .../resources/metrics/MetricsResource.java | 2 +- .../resources/mlmodels/MlModelResource.java | 2 +- .../resources/pipelines/PipelineResource.java | 2 +- .../dashboard/DashboardServiceResource.java | 4 +- .../database/DatabaseServiceResource.java | 2 +- .../messaging/MessagingServiceResource.java | 4 +- .../mlmodel/MlModelServiceResource.java | 2 +- .../pipeline/PipelineServiceResource.java | 2 +- .../storage/StorageServiceResource.java | 2 +- .../resources/storages/ContainerResource.java | 2 +- .../service/resources/teams/TeamResource.java | 5 +- .../service/resources/teams/UserResource.java | 2 +- .../resources/topics/TopicResource.java | 2 +- .../service/resources/EntityResourceTest.java | 29 +- .../automations/WorkflowResourceTest.java | 1 - .../domains/DataProductResourceTest.java | 99 +++++ .../resources/domains/DomainResourceTest.java | 9 + .../events/EventSubscriptionResourceTest.java | 1 - .../glossary/GlossaryResourceTest.java | 16 +- .../org/openmetadata/schema/CreateEntity.java | 14 + .../openmetadata/schema/EntityInterface.java | 17 + .../json/schema/api/data/createChart.json | 11 + .../json/schema/api/data/createContainer.json | 4 + .../json/schema/api/data/createDashboard.json | 11 + .../api/data/createDashboardDataModel.json | 4 + .../json/schema/api/data/createDatabase.json | 4 + .../schema/api/data/createDatabaseSchema.json | 4 + .../json/schema/api/data/createGlossary.json | 4 + .../json/schema/api/data/createMlModel.json | 4 + .../json/schema/api/data/createPipeline.json | 4 + .../json/schema/api/data/createTable.json | 11 + .../json/schema/api/data/createTopic.json | 12 + .../schema/api/domains/createDataProduct.json | 47 +++ .../json/schema/api/domains/createDomain.json | 2 +- .../api/services/createDashboardService.json | 4 + .../api/services/createDatabaseService.json | 4 + .../api/services/createMessagingService.json | 4 + .../api/services/createMlModelService.json | 4 + .../api/services/createPipelineService.json | 4 + .../api/services/createStorageService.json | 4 + .../json/schema/api/teams/createTeam.json | 4 + .../json/schema/entity/data/chart.json | 8 + .../json/schema/entity/data/container.json | 4 + .../json/schema/entity/data/dashboard.json | 8 + .../entity/data/dashboardDataModel.json | 4 + .../json/schema/entity/data/database.json | 4 + .../schema/entity/data/databaseSchema.json | 4 + .../json/schema/entity/data/glossary.json | 4 + .../json/schema/entity/data/glossaryTerm.json | 4 + .../json/schema/entity/data/metrics.json | 4 + .../json/schema/entity/data/mlmodel.json | 5 + .../json/schema/entity/data/pipeline.json | 5 + .../json/schema/entity/data/table.json | 8 + .../json/schema/entity/data/topic.json | 8 + .../schema/entity/domains/dataProduct.json | 66 ++++ .../json/schema/entity/domains/domain.json | 2 +- .../entity/services/dashboardService.json | 4 + .../entity/services/databaseService.json | 4 + .../entity/services/messagingService.json | 4 + .../entity/services/mlmodelService.json | 4 + .../entity/services/pipelineService.json | 4 + .../entity/services/storageService.json | 4 + .../json/schema/entity/teams/team.json | 4 + .../json/schema/entity/teams/user.json | 4 + 83 files changed, 1150 insertions(+), 60 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java create mode 100644 openmetadata-spec/src/main/resources/json/schema/api/domains/createDataProduct.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/domains/dataProduct.json diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql index 6a448d8e558..b34f60ac26a 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql @@ -8,4 +8,16 @@ CREATE TABLE IF NOT EXISTS domain_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, PRIMARY KEY (id), UNIQUE (fqnHash) -); \ No newline at end of file +); + +-- create data product entity table +CREATE TABLE IF NOT EXISTS data_product_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, + fqnHash VARCHAR(256) NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + PRIMARY KEY (id), + UNIQUE (fqnHash) +); diff --git a/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql index 520f2f2885f..77c106aa391 100644 --- a/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql @@ -8,4 +8,16 @@ CREATE TABLE IF NOT EXISTS domain_entity ( updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, PRIMARY KEY (id), UNIQUE (fqnHash) -); \ No newline at end of file +); + +-- create data product entity table +CREATE TABLE IF NOT EXISTS data_product_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, + fqnHash VARCHAR(256) NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + PRIMARY KEY (id), + UNIQUE (fqnHash) +); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 238af16b792..34cdb949e7f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -64,6 +64,8 @@ public final class Entity { public static final String FIELD_DISPLAY_NAME = "displayName"; public static final String FIELD_EXTENSION = "extension"; public static final String FIELD_USAGE_SUMMARY = "usageSummary"; + public static final String FIELD_DOMAIN = "domain"; + public static final String FIELD_DATA_PRODUCTS = "dataProducts"; // // Service entities @@ -132,6 +134,7 @@ public final class Entity { // Domain related entities // public static final String DOMAIN = "domain"; + public static final String DATA_PRODUCT = "dataProduct"; // // Reserved names in OpenMetadata diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index b025f005cf7..967dcb92d5d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -81,6 +81,7 @@ import org.openmetadata.schema.entity.data.Query; import org.openmetadata.schema.entity.data.Report; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.data.Topic; +import org.openmetadata.schema.entity.domains.DataProduct; import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.entity.events.EventSubscription; import org.openmetadata.schema.entity.policies.Policy; @@ -198,6 +199,9 @@ public interface CollectionDAO { @CreateSqlObject DomainDAO domainDAO(); + @CreateSqlObject + DataProductDAO dataProductDAO(); + @CreateSqlObject EventSubscriptionDAO eventSubscriptionDAO(); @@ -1378,6 +1382,28 @@ public interface CollectionDAO { } } + interface DataProductDAO extends EntityDAO { + @Override + default String getTableName() { + return "data_product_entity"; + } + + @Override + default Class getEntityClass() { + return DataProduct.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default boolean supportsSoftDelete() { + return false; + } + } + interface EventSubscriptionDAO extends EntityDAO { @Override default String getTableName() { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java new file mode 100644 index 00000000000..33c497dbd94 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java @@ -0,0 +1,126 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; + +import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.IOException; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.entity.domains.DataProduct; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; +import org.openmetadata.service.resources.domains.DataProductResource; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.EntityUtil.Fields; +import org.openmetadata.service.util.FullyQualifiedName; + +@Slf4j +public class DataProductRepository extends EntityRepository { + private static final String UPDATE_FIELDS = "domain,owner,experts"; // Domain field can't be updated + + public DataProductRepository(CollectionDAO dao) { + super( + DataProductResource.COLLECTION_PATH, + Entity.DATA_PRODUCT, + DataProduct.class, + dao.dataProductDAO(), + dao, + UPDATE_FIELDS, + UPDATE_FIELDS); + } + + @Override + public DataProduct setFields(DataProduct entity, Fields fields) throws IOException { + return entity.withExperts(fields.contains("experts") ? getExperts(entity) : null); + } + + // TODO to to inheritance for experts + private List getExperts(DataProduct entity) throws IOException { + List ids = findTo(entity.getId(), Entity.DATA_PRODUCT, Relationship.EXPERT, Entity.USER); + return EntityUtil.populateEntityReferences(ids, Entity.USER); + } + + @Override + public void prepare(DataProduct entity) throws IOException { + // Parent, Experts, Owner are already validated + } + + @Override + public void storeEntity(DataProduct entity, boolean update) throws IOException { + EntityReference domain = entity.getDomain(); + List experts = entity.getExperts(); + entity.withDomain(null).withExperts(null); + store(entity, update); + entity.withDomain(domain).withExperts(experts); + } + + @Override + public void storeRelationships(DataProduct entity) { + addRelationship( + entity.getDomain().getId(), entity.getId(), Entity.DOMAIN, Entity.DATA_PRODUCT, Relationship.CONTAINS); + for (EntityReference expert : listOrEmpty(entity.getExperts())) { + addRelationship(entity.getId(), expert.getId(), Entity.DATA_PRODUCT, Entity.USER, Relationship.EXPERT); + } + } + + @Override + public EntityUpdater getUpdater(DataProduct original, DataProduct updated, Operation operation) { + return new DataProductUpdater(original, updated, operation); + } + + @Override + public void restorePatchAttributes(DataProduct original, DataProduct updated) { + updated.withDomain(original.getDomain()); // Domain can't be changed + } + + @Override + public void setFullyQualifiedName(DataProduct entity) { + EntityReference domain = entity.getDomain(); + entity.setFullyQualifiedName(FullyQualifiedName.add(domain.getFullyQualifiedName(), entity.getName())); + } + + @Override + public String getFullyQualifiedNameHash(DataProduct entity) { + return FullyQualifiedName.buildHash(entity.getFullyQualifiedName()); + } + + public class DataProductUpdater extends EntityUpdater { + public DataProductUpdater(DataProduct original, DataProduct updated, Operation operation) { + super(original, updated, operation); + } + + @Override + public void entitySpecificUpdate() throws IOException { + updateExperts(); + } + + private void updateExperts() throws JsonProcessingException { + List origExperts = listOrEmpty(original.getExperts()); + List updatedExperts = listOrEmpty(updated.getExperts()); + updateToRelationships( + "experts", + Entity.DATA_PRODUCT, + original.getId(), + Relationship.EXPERT, + Entity.USER, + origExperts, + updatedExperts, + false); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java index 3db0a3f6b73..d097f60f248 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java @@ -33,7 +33,7 @@ import org.openmetadata.service.util.FullyQualifiedName; @Slf4j public class DomainRepository extends EntityRepository { - private static final String UPDATE_FIELDS = "owner,experts"; + private static final String UPDATE_FIELDS = "parent,children,owner,experts"; public DomainRepository(CollectionDAO dao) { super( @@ -50,8 +50,7 @@ public class DomainRepository extends EntityRepository { public Domain setFields(Domain entity, Fields fields) throws IOException { entity.withParent(fields.contains("parent") ? getParent(entity) : null); entity.withChildren(fields.contains("children") ? getChildren(entity) : null); - entity.withExperts(fields.contains("experts") ? getExperts(entity) : null); - return entity.withOwner(fields.contains("owner") ? getOwner(entity) : null); + return entity.withExperts(fields.contains("experts") ? getExperts(entity) : null); } private EntityReference getParent(Domain entity) throws IOException { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index a852f44c18c..00cc8777b50 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -19,9 +19,13 @@ import static org.openmetadata.schema.type.Include.ALL; import static org.openmetadata.schema.type.Include.DELETED; import static org.openmetadata.schema.type.Include.NON_DELETED; import static org.openmetadata.service.Entity.ADMIN_USER_NAME; +import static org.openmetadata.service.Entity.DATA_PRODUCT; +import static org.openmetadata.service.Entity.DOMAIN; +import static org.openmetadata.service.Entity.FIELD_DATA_PRODUCTS; import static org.openmetadata.service.Entity.FIELD_DELETED; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; +import static org.openmetadata.service.Entity.FIELD_DOMAIN; import static org.openmetadata.service.Entity.FIELD_EXTENSION; import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; import static org.openmetadata.service.Entity.FIELD_OWNER; @@ -163,6 +167,8 @@ public abstract class EntityRepository { @Getter protected final boolean supportsOwner; protected final boolean supportsFollower; protected final boolean supportsVotes; + protected final boolean supportsDomain; + protected final boolean supportsDataProducts; /** Fields that can be updated during PATCH operation */ @Getter private final Fields patchFields; @@ -192,6 +198,8 @@ public abstract class EntityRepository { this.supportsSoftDelete = allowedFields.contains(FIELD_DELETED); this.supportsFollower = allowedFields.contains(FIELD_FOLLOWERS); this.supportsVotes = allowedFields.contains(FIELD_VOTES); + this.supportsDomain = allowedFields.contains(FIELD_DOMAIN); + this.supportsDataProducts = allowedFields.contains(FIELD_DATA_PRODUCTS); } /** @@ -547,11 +555,14 @@ public abstract class EntityRepository { prepare(entity); setFullyQualifiedName(entity); validateExtension(entity); + // Domain is already validated } public void storeRelationshipsInternal(T entity) throws IOException { storeOwner(entity, entity.getOwner()); applyTags(entity); + storeDomain(entity, entity.getDomain()); + storeDataProducts(entity, entity.getDataProducts()); storeRelationships(entity); } @@ -559,6 +570,8 @@ public abstract class EntityRepository { entity.setOwner(fields.contains(FIELD_OWNER) ? getOwner(entity) : null); entity.setTags(fields.contains(FIELD_TAGS) ? getTags(entity.getFullyQualifiedName()) : null); entity.setExtension(fields.contains(FIELD_EXTENSION) ? getExtension(entity) : null); + entity.setDomain(fields.contains(FIELD_DOMAIN) ? getDomain(entity) : null); + entity.setDataProducts(fields.contains(FIELD_DATA_PRODUCTS) ? getDataProducts(entity) : null); setFields(entity, fields); setInheritedFields(entity); return entity; @@ -901,6 +914,10 @@ public abstract class EntityRepository { entity.setOwner(null); List tags = entity.getTags(); entity.setTags(null); + EntityReference domain = entity.getDomain(); + entity.setDomain(null); + List dataProducts = entity.getDataProducts(); + entity.setDataProducts(null); if (update) { dao.update(entity.getId(), getFullyQualifiedNameHash(entity), JsonUtils.pojoToJson(entity)); @@ -913,6 +930,8 @@ public abstract class EntityRepository { // Restore the relationships entity.setOwner(owner); entity.setTags(tags); + entity.setDomain(domain); + entity.setDataProducts(dataProducts); } protected void storeTimeSeries( @@ -990,7 +1009,7 @@ public abstract class EntityRepository { daoCollection.entityExtensionTimeSeriesDao().deleteBeforeTimestamp(fqnHash, extension, timestamp); } - public void validateExtension(T entity) { + private void validateExtension(T entity) { if (entity.getExtension() == null) { return; } @@ -1357,6 +1376,18 @@ public abstract class EntityRepository { return !supportsOwner ? null : getFromEntityRef(entity.getId(), Relationship.OWNS, null, false); } + public EntityReference getDomain(T entity) throws IOException { + return getFromEntityRef(entity.getId(), Relationship.HAS, DOMAIN, false); + } + + private List getDataProducts(T entity) throws IOException { + if (!supportsDataProducts || entity == null) { + return null; + } + List ids = findFrom(entity.getId(), entityType, Relationship.HAS, DATA_PRODUCT); + return EntityUtil.populateEntityReferences(ids, entityType); + } + public EntityReference getOwner(EntityReference ref) throws IOException { return !supportsOwner ? null : Entity.getEntityReferenceById(ref.getType(), ref.getId(), ALL); } @@ -1372,15 +1403,39 @@ public abstract class EntityRepository { protected void storeOwner(T entity, EntityReference owner) { if (supportsOwner && owner != null) { // Add relationship owner --- owns ---> ownedEntity - LOG.info("Adding owner {}:{} for entity {}:{}", owner.getType(), owner.getId(), entityType, entity.getId()); + LOG.info( + "Adding owner {}:{} for entity {}:{}", + owner.getType(), + owner.getFullyQualifiedName(), + entityType, + entity.getId()); addRelationship(owner.getId(), entity.getId(), owner.getType(), entityType, Relationship.OWNS); } } + protected void storeDomain(T entity, EntityReference domain) { + if (supportsDomain && domain != null) { + // Add relationship domain --- has ---> entity + LOG.info("Adding domain {} for entity {}:{}", domain.getFullyQualifiedName(), entityType, entity.getId()); + addRelationship(domain.getId(), entity.getId(), Entity.DOMAIN, entityType, Relationship.HAS); + } + } + + protected void storeDataProducts(T entity, List dataProducts) { + if (supportsDataProducts && !nullOrEmpty(dataProducts)) { + for (EntityReference dataProduct : dataProducts) { + // Add relationship dataProduct --- has ---> entity + LOG.info( + "Adding dataProduct {} for entity {}:{}", dataProduct.getFullyQualifiedName(), entityType, entity.getId()); + addRelationship(dataProduct.getId(), entity.getId(), Entity.DATA_PRODUCT, entityType, Relationship.HAS); + } + } + } + /** Remove owner relationship for a given entity */ private void removeOwner(T entity, EntityReference owner) { if (EntityUtil.getId(owner) != null) { - LOG.info("Removing owner {}:{} for entity {}", owner.getType(), owner.getId(), entity.getId()); + LOG.info("Removing owner {}:{} for entity {}", owner.getType(), owner.getFullyQualifiedName(), entity.getId()); deleteRelationship(owner.getId(), owner.getType(), entity.getId(), entityType, Relationship.OWNS); } } @@ -1457,6 +1512,13 @@ public abstract class EntityRepository { return Entity.getEntityReferenceById(owner.getType(), owner.getId(), ALL); } + public EntityReference validateDomain(String domainFqn) throws IOException { + if (!supportsDomain || domainFqn == null) { + return null; + } + return Entity.getEntityReferenceByName(Entity.DOMAIN, domainFqn, NON_DELETED); + } + /** Override this method to support downloading CSV functionality */ public String exportToCsv(String name, String user) throws IOException { throw new IllegalArgumentException(csvNotSupported(entityType)); @@ -1531,6 +1593,8 @@ public abstract class EntityRepository { updateOwner(); updateExtension(); updateTags(updated.getFullyQualifiedName(), FIELD_TAGS, original.getTags(), updated.getTags()); + updateDomain(); + updateDataProducts(); entitySpecificUpdate(); } @@ -1664,6 +1728,47 @@ public abstract class EntityRepository { storeExtension(updated); } + private void updateDomain() throws JsonProcessingException { + if (original.getDomain() == updated.getDomain()) { + return; + } + + EntityReference origDomain = original.getDomain(); + EntityReference updatedDomain = updated.getDomain(); + if ((operation.isPatch() || updatedDomain != null) + && recordChange(FIELD_DOMAIN, origDomain, updatedDomain, true, entityReferenceMatch)) { + if (origDomain != null) { + LOG.info( + "Removing domain {} for entity {}", origDomain.getFullyQualifiedName(), original.getFullyQualifiedName()); + deleteRelationship(origDomain.getId(), Entity.DOMAIN, original.getId(), entityType, Relationship.HAS); + } + if (updatedDomain != null) { + // Add relationship owner --- owns ---> ownedEntity + LOG.info( + "Adding domain {} for entity {}", + updatedDomain.getFullyQualifiedName(), + original.getFullyQualifiedName()); + addRelationship(updatedDomain.getId(), original.getId(), Entity.DOMAIN, entityType, Relationship.HAS); + } + } + } + + private void updateDataProducts() throws JsonProcessingException { + if (!supportsDataProducts) { + return; + } + List origDataProducts = listOrEmpty(original.getDataProducts()); + List updatedDataProducts = listOrEmpty(updated.getDataProducts()); + updateFromRelationships( + "dataProducts", + DATA_PRODUCT, + origDataProducts, + updatedDataProducts, + Relationship.HAS, + entityType, + original.getId()); + } + public final boolean updateVersion(Double oldVersion) { Double newVersion = oldVersion; if (majorVersionChange) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java index 96e61b3a930..7d88ebaa5ed 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java @@ -217,7 +217,6 @@ public abstract class EntityResource { public static final String COLLECTION_PATH = "v1/charts/"; - static final String FIELDS = "owner,followers,tags"; + static final String FIELDS = "owner,followers,tags,domain,dataProducts"; @Override public Chart addHref(UriInfo uriInfo, Chart chart) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java index e569df0e68a..209aa812085 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java @@ -73,7 +73,8 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "dashboards") public class DashboardResource extends EntityResource { public static final String COLLECTION_PATH = "v1/dashboards/"; - protected static final String FIELDS = "owner,charts,followers,tags,usageSummary,extension,dataModels"; + protected static final String FIELDS = + "owner,charts,followers,tags,usageSummary,extension,dataModels," + "domain,dataProducts"; @Override public Dashboard addHref(UriInfo uriInfo, Dashboard dashboard) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java index 89cb924f54f..2ecbae77b66 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java @@ -71,7 +71,7 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "databases") public class DatabaseResource extends EntityResource { public static final String COLLECTION_PATH = "v1/databases/"; - static final String FIELDS = "owner,databaseSchemas,usageSummary,location,tags,extension"; + static final String FIELDS = "owner,databaseSchemas,usageSummary,location,tags,extension,domain"; @Override public Database addHref(UriInfo uriInfo, Database db) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java index ea2a7f9beaf..e1befc84335 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java @@ -71,7 +71,7 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "databaseSchemas") public class DatabaseSchemaResource extends EntityResource { public static final String COLLECTION_PATH = "v1/databaseSchemas/"; - static final String FIELDS = "owner,tables,usageSummary,tags,extension"; + static final String FIELDS = "owner,tables,usageSummary,tags,extension,domain"; @Override public DatabaseSchema addHref(UriInfo uriInfo, DatabaseSchema schema) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index e65ca1f4d5d..788659b5b62 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -84,6 +84,9 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "tables") public class TableResource extends EntityResource { public static final String COLLECTION_PATH = "v1/tables/"; + static final String FIELDS = + "tableConstraints,tablePartition,usageSummary,owner,customMetrics," + + "tags,followers,joins,viewDefinition,dataModel,extension,testSuite,domain,dataProducts"; @Override public Table addHref(UriInfo uriInfo, Table table) { @@ -136,10 +139,6 @@ public class TableResource extends EntityResource { /* Required for serde */ } - static final String FIELDS = - "tableConstraints,tablePartition,usageSummary,owner,customMetrics," - + "tags,followers,joins,viewDefinition,dataModel,extension,testSuite"; - @GET @Operation( operationId = "listTables", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java index 9a0a1774c9d..2ba5640915b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java @@ -70,7 +70,7 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "datamodels") public class DashboardDataModelResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/dashboard/datamodels"; - protected static final String FIELDS = "owner,tags,followers"; + protected static final String FIELDS = "owner,tags,followers,domain"; @Override public DashboardDataModel addHref(UriInfo uriInfo, DashboardDataModel dashboardDataModel) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java new file mode 100644 index 00000000000..f5d4465f4ec --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java @@ -0,0 +1,341 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.resources.domains; + +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.io.IOException; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; +import javax.json.JsonPatch; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.PATCH; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.api.domains.CreateDataProduct; +import org.openmetadata.schema.entity.domains.DataProduct; +import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.utils.EntityInterfaceUtil; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.DataProductRepository; +import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.resources.EntityResource; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.RestUtil; +import org.openmetadata.service.util.ResultList; + +@Slf4j +@Path("/v1/dataProducts") +@Tag( + name = "Domains", + description = + "A `Data Product` or `Data as a Product` is a logical unit that contains all components to process and store " + + "domain data for analytical or data-intensive use cases made available to data consumers.") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "dataProducts", order = 4) // initialize after user resource +public class DataProductResource extends EntityResource { + public static final String COLLECTION_PATH = "/v1/dataProducts/"; + static final String FIELDS = "domain,owner,experts"; + + public DataProductResource(CollectionDAO dao, Authorizer authorizer) { + super(DataProduct.class, new DataProductRepository(dao), authorizer); + } + + @Override + public DataProduct addHref(UriInfo uriInfo, DataProduct dataProduct) { + dataProduct.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, dataProduct.getId())); + Entity.withHref(uriInfo, dataProduct.getExperts()); + Entity.withHref(uriInfo, dataProduct.getOwner()); + Entity.withHref(uriInfo, dataProduct.getDomain()); + return dataProduct; + } + + public static class DataProductList extends ResultList { + @SuppressWarnings("unused") + public DataProductList() { + /* Required for serde */ + } + } + + @GET + @Operation( + operationId = "listDataProducts", + summary = "List dataProducts", + description = "Get a list of DataProducts.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of DataProducts", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = DataProductList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam, + @Parameter(description = "Returns list of DataProduct before this cursor", schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter(description = "Returns list of DataProduct after this cursor", schema = @Schema(type = "string")) + @QueryParam("after") + String after) + throws IOException { + return listInternal(uriInfo, securityContext, fieldsParam, new ListFilter(null), limitParam, before, after); + } + + @GET + @Path("/{id}") + @Operation( + operationId = "getDataProductByID", + summary = "Get a dataProduct by Id", + description = "Get a dataProduct by `Id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The dataProduct", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = DataProduct.class))), + @ApiResponse(responseCode = "404", description = "DataProduct for instance {id} is not found") + }) + public DataProduct get( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter(description = "Id of the dataProduct", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) + throws IOException { + return getInternal(uriInfo, securityContext, id, fieldsParam, null); + } + + @GET + @Path("/name/{name}") + @Operation( + operationId = "getDataProductByFQN", + summary = "Get a dataProduct by name", + description = "Get a dataProduct by `name`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "dataProduct", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = DataProduct.class))), + @ApiResponse(responseCode = "404", description = "DataProduct for instance {name} is not found") + }) + public DataProduct getByName( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Name of the dataProduct", schema = @Schema(type = "string")) @PathParam("name") + String name, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam) + throws IOException { + return getByNameInternal(uriInfo, securityContext, name, fieldsParam, null); + } + + @GET + @Path("/{id}/versions") + @Operation( + operationId = "listAllDataProductVersion", + summary = "List dataProduct versions", + description = "Get a list of all the versions of a dataProduct identified by `Id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of dataProduct versions", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class))) + }) + public EntityHistory listVersions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the dataProduct", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) + throws IOException { + return super.listVersionsInternal(securityContext, id); + } + + @GET + @Path("/{id}/versions/{version}") + @Operation( + operationId = "listSpecificDataProductVersion", + summary = "Get a version of the dataProduct", + description = "Get a version of the dataProduct by given `Id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "dataProduct", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = DataProduct.class))), + @ApiResponse( + responseCode = "404", + description = "DataProduct for instance {id} and version {version} is " + "not found") + }) + public DataProduct getVersion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the dataProduct", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Parameter( + description = "DataProduct version number in the form `major`.`minor`", + schema = @Schema(type = "string", example = "0.1 or 1.1")) + @PathParam("version") + String version) + throws IOException { + return super.getVersionInternal(securityContext, id, version); + } + + @POST + @Operation( + operationId = "createDataProduct", + summary = "Create a dataProduct", + description = "Create a new dataProduct.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The dataProduct ", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = DataProduct.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response create( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDataProduct create) + throws IOException { + DataProduct dataProduct = getDataProduct(create, securityContext.getUserPrincipal().getName()); + return create(uriInfo, securityContext, dataProduct); + } + + @PUT + @Operation( + operationId = "createOrUpdateDataProduct", + summary = "Create or update a dataProduct", + description = + "Create a dataProduct. if it does not exist. If a dataProduct already exists, update the " + "dataProduct.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The dataProduct", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = DataProduct.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response createOrUpdate( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDataProduct create) + throws IOException { + DataProduct dataProduct = getDataProduct(create, securityContext.getUserPrincipal().getName()); + return createOrUpdate(uriInfo, securityContext, dataProduct); + } + + @PATCH + @Path("/{id}") + @Operation( + operationId = "patchDataProduct", + summary = "Update a dataProduct", + description = "Update an existing dataProduct using JsonPatch.", + externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902")) + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + public Response patch( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the dataProduct", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @RequestBody( + description = "JsonPatch with array of operations", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON_PATCH_JSON, + examples = { + @ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]") + })) + JsonPatch patch) + throws IOException { + return patchInternal(uriInfo, securityContext, id, patch); + } + + @DELETE + @Path("/{id}") + @Operation( + operationId = "deleteDataProduct", + summary = "Delete a dataProduct by Id", + description = "Delete a dataProduct by `Id`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "DataProduct for instance {id} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the dataProduct", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) + throws IOException { + return delete(uriInfo, securityContext, id, true, true); + } + + @DELETE + @Path("/name/{name}") + @Operation( + operationId = "deleteDataProductByFQN", + summary = "Delete a dataProduct by name", + description = "Delete a dataProduct by `name`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "DataProduct for instance {name} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Name of the dataProduct", schema = @Schema(type = "string")) @PathParam("name") + String name) + throws IOException { + return deleteByName(uriInfo, securityContext, name, true, true); + } + + private DataProduct getDataProduct(CreateDataProduct create, String user) throws IOException { + List experts = + create.getExperts() == null + ? create.getExperts() + : create.getExperts().stream().map(EntityInterfaceUtil::quoteName).collect(Collectors.toList()); + return copy(new DataProduct(), create, user) + .withFullyQualifiedName(create.getName()) + .withExperts(EntityUtil.populateEntityReferences(getEntityReferences(Entity.USER, experts))); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java index cfc236fcabd..c38e8d32a5f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java @@ -50,6 +50,7 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.domains.CreateDomain; import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.type.Include; import org.openmetadata.schema.utils.EntityInterfaceUtil; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; @@ -332,7 +333,8 @@ public class DomainResource extends EntityResource { return copy(new Domain(), create, user) .withDomainType(create.getDomainType()) .withFullyQualifiedName(create.getName()) - .withParent(getEntityReference(Entity.DOMAIN, create.getParent())) + .withParent( + Entity.getEntityReference(getEntityReference(Entity.DOMAIN, create.getParent()), Include.NON_DELETED)) .withExperts(EntityUtil.populateEntityReferences(getEntityReferences(Entity.USER, experts))); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index b125932fffe..423711f3018 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -339,7 +339,6 @@ public class TestCaseResource extends EntityResource { public static final String COLLECTION_PATH = "v1/glossaries/"; - static final String FIELDS = "owner,tags,reviewers,usageCount,termCount"; + static final String FIELDS = "owner,tags,reviewers,usageCount,termCount,domain"; @Override public Glossary addHref(UriInfo uriInfo, Glossary glossary) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java index 4085a8ca028..6436519165f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java @@ -76,7 +76,7 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "glossaryTerms", order = 7) // Initialized after Glossary, Classification, and Tags public class GlossaryTermResource extends EntityResource { public static final String COLLECTION_PATH = "v1/glossaryTerms/"; - static final String FIELDS = "children,relatedTerms,reviewers,owner,tags,usageCount"; + static final String FIELDS = "children,relatedTerms,reviewers,owner,tags,usageCount,domain"; @Override public GlossaryTerm addHref(UriInfo uriInfo, GlossaryTerm term) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricsResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricsResource.java index 1af3484ec86..f13bab9504f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricsResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricsResource.java @@ -63,7 +63,7 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "metrics") public class MetricsResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/metrics/"; - static final String FIELDS = "owner,usageSummary"; + static final String FIELDS = "owner,usageSummary,domain"; public MetricsResource(CollectionDAO dao, Authorizer authorizer) { super(Metrics.class, new MetricsRepository(dao), authorizer); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java index fd701d24373..82ca8b1d816 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java @@ -73,7 +73,7 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "mlmodels") public class MlModelResource extends EntityResource { public static final String COLLECTION_PATH = "v1/mlmodels/"; - static final String FIELDS = "owner,dashboard,followers,tags,usageSummary,extension"; + static final String FIELDS = "owner,dashboard,followers,tags,usageSummary,extension,domain"; @Override public MlModel addHref(UriInfo uriInfo, MlModel mlmodel) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java index 78060429e16..cbb0205b863 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java @@ -78,7 +78,7 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "pipelines") public class PipelineResource extends EntityResource { public static final String COLLECTION_PATH = "v1/pipelines/"; - static final String FIELDS = "owner,tasks,pipelineStatus,followers,tags,extension,scheduleInterval"; + static final String FIELDS = "owner,tasks,pipelineStatus,followers,tags,extension,scheduleInterval,domain"; @Override public Pipeline addHref(UriInfo uriInfo, Pipeline pipeline) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceResource.java index 2de355c64c6..4e8e87a1fc4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceResource.java @@ -13,8 +13,6 @@ package org.openmetadata.service.resources.services.dashboard; -import static org.openmetadata.service.Entity.FIELD_OWNER; - import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -79,7 +77,7 @@ import org.openmetadata.service.util.ResultList; public class DashboardServiceResource extends ServiceEntityResource { public static final String COLLECTION_PATH = "v1/services/dashboardServices"; - static final String FIELDS = FIELD_OWNER; + static final String FIELDS = "owner,domain"; @Override public DashboardService addHref(UriInfo uriInfo, DashboardService service) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java index 1aef41f4a56..25b717d7ca7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java @@ -83,7 +83,7 @@ import org.openmetadata.service.util.ResultList; public class DatabaseServiceResource extends ServiceEntityResource { public static final String COLLECTION_PATH = "v1/services/databaseServices/"; - static final String FIELDS = "pipelines,owner,tags"; + static final String FIELDS = "pipelines,owner,tags,domain"; @Override public DatabaseService addHref(UriInfo uriInfo, DatabaseService service) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceResource.java index 9b331445d2e..74789025058 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceResource.java @@ -13,8 +13,6 @@ package org.openmetadata.service.resources.services.messaging; -import static org.openmetadata.service.Entity.FIELD_OWNER; - import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -79,7 +77,7 @@ import org.openmetadata.service.util.ResultList; public class MessagingServiceResource extends ServiceEntityResource { public static final String COLLECTION_PATH = "v1/services/messagingServices/"; - public static final String FIELDS = FIELD_OWNER; + public static final String FIELDS = "owner,domain"; @Override public MessagingService addHref(UriInfo uriInfo, MessagingService service) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceResource.java index ad45e5eb4e7..5cdc6e9d635 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceResource.java @@ -79,7 +79,7 @@ import org.openmetadata.service.util.ResultList; public class MlModelServiceResource extends ServiceEntityResource { public static final String COLLECTION_PATH = "v1/services/mlmodelServices/"; - public static final String FIELDS = "pipelines,owner,tags"; + public static final String FIELDS = "pipelines,owner,tags,domain"; @Override public MlModelService addHref(UriInfo uriInfo, MlModelService service) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceResource.java index 80a72070589..12d909e8a21 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceResource.java @@ -77,7 +77,7 @@ import org.openmetadata.service.util.ResultList; public class PipelineServiceResource extends ServiceEntityResource { public static final String COLLECTION_PATH = "v1/services/pipelineServices/"; - static final String FIELDS = "pipelines,owner"; + static final String FIELDS = "pipelines,owner,domain"; @Override public PipelineService addHref(UriInfo uriInfo, PipelineService service) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceResource.java index 8c6893e78b2..44464555cdb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceResource.java @@ -69,7 +69,7 @@ import org.openmetadata.service.util.ResultList; public class StorageServiceResource extends ServiceEntityResource { public static final String COLLECTION_PATH = "v1/services/storageServices/"; - static final String FIELDS = "pipelines,owner,tags"; + static final String FIELDS = "pipelines,owner,tags,domain"; @Override public StorageService addHref(UriInfo uriInfo, StorageService service) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerResource.java index 1700db66d5b..0a33ea1b49f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerResource.java @@ -61,7 +61,7 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "containers") public class ContainerResource extends EntityResource { public static final String COLLECTION_PATH = "v1/containers/"; - static final String FIELDS = "parent,children,dataModel,owner,tags,followers,extension"; + static final String FIELDS = "parent,children,dataModel,owner,tags,followers,extension,domain"; @Override public Container addHref(UriInfo uriInfo, Container container) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java index 6b449948a56..28fb2ef73ab 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java @@ -85,6 +85,8 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "teams", order = 2) // Load after roles, and policy resources public class TeamResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/teams/"; + static final String FIELDS = + "owner,profile,users,owns,defaultRoles,parents,children,policies,userCount,childrenCount,domain"; @Override public Team addHref(UriInfo uriInfo, Team team) { @@ -122,9 +124,6 @@ public class TeamResource extends EntityResource { /* Required for serde */ } - static final String FIELDS = - "owner,profile,users,owns,defaultRoles,parents,children,policies,userCount,childrenCount"; - @GET @Path("/hierarchy") @Valid diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java index 103a4572f57..144380ae7c5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java @@ -159,7 +159,7 @@ public class UserResource extends EntityResource { private boolean isEmailServiceEnabled; private AuthenticationConfiguration authenticationConfiguration; private final AuthenticatorHandler authHandler; - static final String FIELDS = "profile,roles,teams,follows,owns"; + static final String FIELDS = "profile,roles,teams,follows,owns,domain"; @Override public User addHref(UriInfo uriInfo, User user) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java index 5d3601c3734..790fbd8a37d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java @@ -77,7 +77,7 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "topics") public class TopicResource extends EntityResource { public static final String COLLECTION_PATH = "v1/topics/"; - static final String FIELDS = "owner,followers,tags,extension"; + static final String FIELDS = "owner,followers,tags,extension,domain,dataProducts"; @Override public Topic addHref(UriInfo uriInfo, Topic topic) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index 0fe5483202d..d9191155de1 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -141,6 +141,8 @@ import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.entity.data.Glossary; import org.openmetadata.schema.entity.data.GlossaryTerm; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.domains.DataProduct; +import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.entity.policies.Policy; import org.openmetadata.schema.entity.policies.accessControl.Rule; import org.openmetadata.schema.entity.services.connections.TestConnectionResult; @@ -172,6 +174,8 @@ import org.openmetadata.service.elasticsearch.ElasticSearchIndexDefinition; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.resources.bots.BotResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; +import org.openmetadata.service.resources.domains.DataProductResourceTest; +import org.openmetadata.service.resources.domains.DomainResourceTest; import org.openmetadata.service.resources.dqtests.TestCaseResourceTest; import org.openmetadata.service.resources.dqtests.TestDefinitionResourceTest; import org.openmetadata.service.resources.dqtests.TestSuiteResourceTest; @@ -328,6 +332,11 @@ public abstract class EntityResourceTest COLUMNS; + public static Domain DOMAIN; + public static Domain SUB_DOMAIN; + public static DataProduct DOMAIN_DATA_PRODUCT; + public static DataProduct SUB_DOMAIN_DATA_PRODUCT; + public static final TestConnectionResult TEST_CONNECTION_RESULT = new TestConnectionResult() .withStatus(TestConnectionResultStatus.SUCCESSFUL) @@ -405,6 +414,8 @@ public abstract class EntityResourceTest backwardPage; boolean foundDeleted = false; do { // For each limit (or page size) - forward scroll till the end - LOG.debug("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after); + LOG.debug( + "Limit {} forward pageCount {} indexInAllTables {} totalRecords {} afterCursor {}", + limit, + pageCount, + indexInAllTables, + totalRecords, + after); forwardPage = listEntities(queryParams, limit, null, after, ADMIN_AUTH_HEADERS); foundDeleted = forwardPage.getData().stream().anyMatch(matchDeleted) || foundDeleted; after = forwardPage.getPaging().getAfter(); @@ -620,7 +637,13 @@ public abstract class EntityResourceTest { + public DataProductResourceTest() { + super(Entity.DATA_PRODUCT, DataProduct.class, DataProductList.class, "dataProducts", DataProductResource.FIELDS); + supportsFieldsQueryParam = false; // TODO + supportsEmptyDescription = false; + } + + public void setupDataProducts(TestInfo test) throws HttpResponseException { + DOMAIN_DATA_PRODUCT = createEntity(createRequest(getEntityName(test)), ADMIN_AUTH_HEADERS); + SUB_DOMAIN_DATA_PRODUCT = + createEntity( + createRequest(getEntityName(test, 1)).withDomain(SUB_DOMAIN.getFullyQualifiedName()), ADMIN_AUTH_HEADERS); + } + + @Override + public CreateDataProduct createRequest(String name) { + return new CreateDataProduct() + .withName(name) + .withDescription(name) + .withDomain(DOMAIN.getFullyQualifiedName()) + .withExperts(listOf(USER1.getFullyQualifiedName())); + } + + @Override + public void validateCreatedEntity( + DataProduct createdEntity, CreateDataProduct request, Map authHeaders) { + // Entity specific validation + assertEquals(request.getDomain(), createdEntity.getDomain().getFullyQualifiedName()); + assertEntityReferenceNames(request.getExperts(), createdEntity.getExperts()); + } + + @Override + public void compareEntities(DataProduct expected, DataProduct updated, Map authHeaders) { + // Entity specific validation + assertReference(expected.getDomain(), updated.getDomain()); + assertEntityReferences(expected.getExperts(), updated.getExperts()); + } + + @Override + public DataProduct validateGetWithDifferentFields(DataProduct dataProduct, boolean byName) + throws HttpResponseException { + DataProduct getDataProduct = + byName + ? getEntityByName(dataProduct.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS) + : getEntity(dataProduct.getId(), null, ADMIN_AUTH_HEADERS); + assertListNull(getDataProduct.getOwner(), getDataProduct.getExperts()); + String fields = "owner,domain,experts"; + getDataProduct = + byName + ? getEntityByName(getDataProduct.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS) + : getEntity(getDataProduct.getId(), fields, ADMIN_AUTH_HEADERS); + // Fields requested are received + assertReference(dataProduct.getDomain(), getDataProduct.getDomain()); + assertEntityReferences(dataProduct.getExperts(), getDataProduct.getExperts()); + + // Checks for other owner, tags, and followers is done in the base class + return getDataProduct; + } + + @Override + public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException { + if (expected == actual) { + return; + } + if (fieldName.startsWith("domain")) { + EntityReference expectedRef = (EntityReference) expected; + EntityReference actualRef = JsonUtils.readValue(actual.toString(), EntityReference.class); + assertEquals(expectedRef.getId(), actualRef.getId()); + } else if (fieldName.startsWith("experts")) { + @SuppressWarnings("unchecked") + List expectedRefs = (List) expected; + List actualRefs = JsonUtils.readObjects(actual.toString(), EntityReference.class); + assertEntityReferences(expectedRefs, actualRefs); + } else { + assertCommonFieldChange(fieldName, expected, actual); + } + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java index a5b4bbc6cc9..e7e343d6746 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.TestInfo; import org.openmetadata.schema.api.domains.CreateDomain; import org.openmetadata.schema.api.domains.CreateDomain.DomainType; import org.openmetadata.schema.entity.domains.Domain; @@ -24,6 +25,13 @@ public class DomainResourceTest extends EntityResourceTest public DomainResourceTest() { super(Entity.DOMAIN, Domain.class, DomainList.class, "domains", DomainResource.FIELDS); supportsFieldsQueryParam = false; // TODO + supportsEmptyDescription = false; + } + + public void setupDomains(TestInfo test) throws IOException { + DOMAIN = createEntity(createRequest(test), ADMIN_AUTH_HEADERS); + SUB_DOMAIN = + createEntity(createRequest("sub-domain").withParent(DOMAIN.getFullyQualifiedName()), ADMIN_AUTH_HEADERS); } @Override @@ -31,6 +39,7 @@ public class DomainResourceTest extends EntityResourceTest return new CreateDomain() .withName(name) .withDomainType(DomainType.AGGREGATE) + .withDescription("name") .withExperts(listOf(USER1.getFullyQualifiedName())); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/events/EventSubscriptionResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/events/EventSubscriptionResourceTest.java index 4368fbd9949..5e203534bb6 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/events/EventSubscriptionResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/events/EventSubscriptionResourceTest.java @@ -59,7 +59,6 @@ public class EventSubscriptionResourceTest extends EntityResourceTest getDataProducts() { + return null; + } + K withName(String name); K withDisplayName(String displayName); @@ -43,4 +53,8 @@ public interface CreateEntity { default K withExtension(Object extension) { return (K) this; } + + default K withDomain(String domain) { + return (K) this; + } } diff --git a/openmetadata-spec/src/main/java/org/openmetadata/schema/EntityInterface.java b/openmetadata-spec/src/main/java/org/openmetadata/schema/EntityInterface.java index bcab7f1f245..7447ea5cacb 100644 --- a/openmetadata-spec/src/main/java/org/openmetadata/schema/EntityInterface.java +++ b/openmetadata-spec/src/main/java/org/openmetadata/schema/EntityInterface.java @@ -28,6 +28,7 @@ import org.openmetadata.schema.type.Votes; import org.openmetadata.schema.utils.EntityInterfaceUtil; /** Interface to be implemented by all entities to provide a way to access all the common fields. */ +@SuppressWarnings("unused") public interface EntityInterface { // Lower case entity name to canonical entity name map Map CANONICAL_ENTITY_NAME_MAP = new HashMap<>(); @@ -81,6 +82,14 @@ public interface EntityInterface { return null; } + default EntityReference getDomain() { + return null; + } + + default List getDataProducts() { + return null; + } + void setId(UUID id); void setDescription(String description); @@ -115,6 +124,14 @@ public interface EntityInterface { /* no-op implementation to be overridden */ } + default void setDomain(EntityReference entityReference) { + /* no-op implementation to be overridden */ + } + + default void setDataProducts(List dataProducts) { + /* no-op implementation to be overridden */ + } + T withHref(URI href); @JsonIgnore diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createChart.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createChart.json index f648dba6fbf..91bd8146e5c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createChart.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createChart.json @@ -41,6 +41,17 @@ "service": { "description": "Link to the chart service where this chart is hosted in", "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "domain" : { + "description": "Fully qualified name of the domain the Chart belongs to.", + "$ref" : "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "dataProducts" : { + "description": "List of fully qualified names of data products this entity is part of.", + "type": "array", + "items" : { + "$ref" : "../../type/basic.json#/definitions/fullyQualifiedEntityName" + } } }, "required": ["name", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createContainer.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createContainer.json index 7415bc0e640..001960f37fa 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createContainer.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createContainer.json @@ -71,6 +71,10 @@ "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Fully qualified name of the domain the Container belongs to.", + "type": "string" } }, "required": ["name", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboard.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboard.json index c247388003d..617c59eb54f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboard.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboard.json @@ -65,6 +65,17 @@ "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Fully qualified name of the domain the Dashboard belongs to.", + "$ref" : "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "dataProducts" : { + "description": "List of fully qualified names of data products this entity is part of.", + "type": "array", + "items" : { + "$ref" : "../../type/basic.json#/definitions/fullyQualifiedEntityName" + } } }, "required": ["name", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboardDataModel.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboardDataModel.json index 5ba60068a32..84020b0e0a9 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboardDataModel.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboardDataModel.json @@ -58,6 +58,10 @@ "project": { "description": "Name of the project / workspace / collection in which the dataModel is contained", "type": "string" + }, + "domain" : { + "description": "Fully qualified name of the domain the Dashboard Data Model belongs to.", + "type": "string" } }, "required": ["name", "service", "dataModelType", "columns"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json index b0c7156e89a..8b932f8eb61 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json @@ -48,6 +48,10 @@ "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Fully qualified name of the domain the Database belongs to.", + "type": "string" } }, "required": ["name", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json index d5d3458912c..412651c149b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json @@ -44,6 +44,10 @@ "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Fully qualified name of the domain the Database Schema belongs to.", + "type": "string" } }, "required": [ diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createGlossary.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createGlossary.json index ac2f8236248..7fcf26f9a24 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createGlossary.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createGlossary.json @@ -46,6 +46,10 @@ "description" : "Glossary terms that are direct children in this glossary are mutually exclusive. When mutually exclusive is `true` only one term can be used to label an entity. When mutually exclusive is `false`, multiple terms from this group can be used to label an entity.", "type" : "boolean", "default" : "false" + }, + "domain" : { + "description": "Fully qualified name of the domain the Glossary belongs to.", + "type": "string" } }, "required": ["name", "description"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createMlModel.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createMlModel.json index 775cbf0af0e..7977202fc43 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createMlModel.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createMlModel.json @@ -75,6 +75,10 @@ "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Fully qualified name of the domain the MLModel belongs to.", + "type": "string" } }, "required": ["name", "algorithm", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createPipeline.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createPipeline.json index 0535d8ee837..48114b539bf 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createPipeline.json @@ -68,6 +68,10 @@ "description": "Scheduler Interval for the pipeline in cron format.", "type": "string", "default": null + }, + "domain" : { + "description": "Fully qualified name of the domain the Pipeline belongs to.", + "type": "string" } }, "required": ["name", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json index 85179dde85d..3299d293532 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json @@ -77,6 +77,17 @@ "sourceUrl": { "description": "Source URL of table.", "$ref": "../../type/basic.json#/definitions/sourceUrl" + }, + "domain" : { + "description": "Fully qualified name of the domain the Table belongs to.", + "type": "string" + }, + "dataProducts" : { + "description": "List of fully qualified names of data products this entity is part of.", + "type": "array", + "items" : { + "$ref" : "../../type/basic.json#/definitions/fullyQualifiedEntityName" + } } }, "required": ["name", "columns", "databaseSchema"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createTopic.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createTopic.json index 65f5ff150ab..6635e4561eb 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createTopic.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createTopic.json @@ -79,6 +79,18 @@ "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Fully qualified name of the domain the Topic belongs to.", + "type": "string", + "$ref" : "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "dataProducts" : { + "description": "List of fully qualified names of data products this entity is part of.", + "type": "array", + "items" : { + "$ref" : "../../type/basic.json#/definitions/fullyQualifiedEntityName" + } } }, "required": ["name", "service", "partitions"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/domains/createDataProduct.json b/openmetadata-spec/src/main/resources/json/schema/api/domains/createDataProduct.json new file mode 100644 index 00000000000..5e9c757827f --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/domains/createDataProduct.json @@ -0,0 +1,47 @@ +{ + "$id": "https://open-metadata.org/schema/entity/domains/createDataProduct.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "createDataProduct", + "description": "Create DataProduct API request", + "type": "object", + "javaType": "org.openmetadata.schema.api.domains.CreateDataProduct", + "javaInterfaces": ["org.openmetadata.schema.CreateEntity"], + "properties": { + "name": { + "description": "A unique name of the DataProduct", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "fullyQualifiedName": { + "description": "FullyQualifiedName of the Domain.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "displayName": { + "description": "Name used for display purposes. Example 'Customer Churn', 'Sentiment Analysis', etc.", + "type": "string" + }, + "description": { + "description": "Description of the DataProduct.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "owner": { + "description": "Owner of this DataProduct.", + "$ref": "../../type/entityReference.json", + "default": null + }, + "domain": { + "description": "Fully qualified name of the Domain the DataProduct belongs to.", + "$ref" : "../../type/basic.json#/definitions/fullyQualifiedEntityName", + "default": null + }, + "experts": { + "description": "List of of user/login names of users who are experts in this DataProduct.", + "type" : "array", + "items": { + "type" : "string" + }, + "default": null + } + }, + "required": ["id", "name", "description", "domain","href"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/api/domains/createDomain.json b/openmetadata-spec/src/main/resources/json/schema/api/domains/createDomain.json index 87ee00935ae..8b6a5cc5a56 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/domains/createDomain.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/domains/createDomain.json @@ -45,6 +45,6 @@ "default": null } }, - "required": ["id", "name", "domainType","href"], + "required": ["id", "name", "description", "domainType","href"], "additionalProperties": false } diff --git a/openmetadata-spec/src/main/resources/json/schema/api/services/createDashboardService.json b/openmetadata-spec/src/main/resources/json/schema/api/services/createDashboardService.json index 287167e760e..c9da120c794 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/services/createDashboardService.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/services/createDashboardService.json @@ -36,6 +36,10 @@ "owner": { "description": "Owner of this dashboard service.", "$ref": "../../type/entityReference.json" + }, + "domain" : { + "description": "Fully qualified name of the domain the Dashboard Service belongs to.", + "type": "string" } }, "required": ["name", "serviceType", "connection"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/services/createDatabaseService.json b/openmetadata-spec/src/main/resources/json/schema/api/services/createDatabaseService.json index 780919998f7..11d1254081b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/services/createDatabaseService.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/services/createDatabaseService.json @@ -37,6 +37,10 @@ "owner": { "description": "Owner of this database service.", "$ref": "../../type/entityReference.json" + }, + "domain" : { + "description": "Fully qualified name of the domain the Database Service belongs to.", + "type": "string" } }, "required": ["name", "serviceType", "connection"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/services/createMessagingService.json b/openmetadata-spec/src/main/resources/json/schema/api/services/createMessagingService.json index 25b1ce6a212..b64c8b10f57 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/services/createMessagingService.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/services/createMessagingService.json @@ -37,6 +37,10 @@ "owner": { "description": "Owner of this messaging service.", "$ref": "../../type/entityReference.json" + }, + "domain" : { + "description": "Fully qualified name of the domain the Messaging Service belongs to.", + "type": "string" } }, "required": ["name", "serviceType", "connection"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/services/createMlModelService.json b/openmetadata-spec/src/main/resources/json/schema/api/services/createMlModelService.json index 92b7ed40494..f1963f98bce 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/services/createMlModelService.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/services/createMlModelService.json @@ -37,6 +37,10 @@ "owner": { "description": "Owner of this mlModel service.", "$ref": "../../type/entityReference.json" + }, + "domain" : { + "description": "Fully qualified name of the domain the MLModel Service belongs to.", + "type": "string" } }, "required": ["name", "serviceType", "connection"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/services/createPipelineService.json b/openmetadata-spec/src/main/resources/json/schema/api/services/createPipelineService.json index bfb30bebec1..050a3988d0e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/services/createPipelineService.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/services/createPipelineService.json @@ -42,6 +42,10 @@ "description": "Scheduler Interval for the pipeline in cron format.", "type": "string", "default": null + }, + "domain" : { + "description": "Fully qualified name of the domain the Pipeline Service belongs to.", + "type": "string" } }, "required": ["name", "serviceType", "connection"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/services/createStorageService.json b/openmetadata-spec/src/main/resources/json/schema/api/services/createStorageService.json index 0d1cab472c1..b9b15fb8994 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/services/createStorageService.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/services/createStorageService.json @@ -37,6 +37,10 @@ "owner": { "description": "Owner of this object store service.", "$ref": "../../type/entityReference.json" + }, + "domain" : { + "description": "Fully qualified name of the domain the Storage Service belongs to.", + "type": "string" } }, "required": ["name", "serviceType", "connection"], diff --git a/openmetadata-spec/src/main/resources/json/schema/api/teams/createTeam.json b/openmetadata-spec/src/main/resources/json/schema/api/teams/createTeam.json index a1d2bcc39ee..bae4ab5c6fb 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/teams/createTeam.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/teams/createTeam.json @@ -80,6 +80,10 @@ "$ref": "../../type/basic.json#/definitions/uuid" }, "default": null + }, + "domain" : { + "description": "Fully qualified name of the domain the Team belongs to.", + "type": "string" } }, "required": ["name", "teamType"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/chart.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/chart.json index 4e6306739ac..8115ee8e238 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/chart.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/chart.json @@ -138,6 +138,14 @@ "description": "When `true` indicates the entity has been soft deleted.", "type": "boolean", "default": false + }, + "domain" : { + "description": "Domain the Chart belongs to. The Chart inherits domain from the dashboard service it belongs to.", + "$ref": "../../type/entityReference.json" + }, + "dataProducts" : { + "description": "List of of data products this entity is part of.", + "$ref" : "../../type/entityReferenceList.json#/definitions/entityReferenceList" } }, "required": ["id", "name", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/container.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/container.json index b826fef3c66..8c1e971ac12 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/container.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/container.json @@ -167,6 +167,10 @@ "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Domain the Container belongs to. When not set, the Container inherits the domain from the storage service it belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": [ diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboard.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboard.json index 5727b157928..11247a9d7db 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboard.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboard.json @@ -126,6 +126,14 @@ "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Domain the Dashboard belongs to. When not set, the Dashboard inherits the domain from the dashboard service it belongs to.", + "$ref": "../../type/entityReference.json" + }, + "dataProducts" : { + "description": "List of of data products this entity is part of.", + "$ref" : "../../type/entityReferenceList.json#/definitions/entityReferenceList" } }, "required": ["id", "name", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json index 38fd8ea4b17..595dc903276 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json @@ -137,6 +137,10 @@ "project": { "description": "Name of the project / workspace / collection in which the dataModel is contained", "type": "string" + }, + "domain" : { + "description": "Domain the Dashboard Data Model belongs to. When not set, the Dashboard model inherits the domain from the dashboard service it belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": [ diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/database.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/database.json index d4757132bbc..af9def04208 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/database.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/database.json @@ -107,6 +107,10 @@ "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Domain the Database belongs to. When not set, the Database inherits the domain from the database service it belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": ["name", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json index 4315310947a..39af79e67d4 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json @@ -102,6 +102,10 @@ "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Domain the Database Schema belongs to. When not set, the Schema inherits the domain from the database it belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": ["name", "database", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/glossary.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/glossary.json index fc9f95beb4f..05d94253b31 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/glossary.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/glossary.json @@ -91,6 +91,10 @@ "description" : "Glossary terms that are direct children in this glossary are mutually exclusive. When mutually exclusive is `true` only one term can be used to label an entity. When mutually exclusive is `false`, multiple terms from this group can be used to label an entity.", "type" : "boolean", "default" : "false" + }, + "domain" : { + "description": "Domain the Glossary belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": ["id", "name", "description"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/glossaryTerm.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/glossaryTerm.json index 2a38b2906d7..5d2034638db 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/glossaryTerm.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/glossaryTerm.json @@ -139,6 +139,10 @@ "description" : "Glossary terms that are children of this term are mutually exclusive. When mutually exclusive is `true` only one term can be used to label an entity from this group. When mutually exclusive is `false`, multiple terms from this group can be used to label an entity.", "type" : "boolean", "default" : "false" + }, + "domain" : { + "description": "Domain the Glossary Term belongs to. When not set, the Glossary TErm inherits the domain from the Glossary it belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": ["id", "name", "description", "glossary"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/metrics.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/metrics.json index bf4dc98ee8e..b4daba9af04 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/metrics.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/metrics.json @@ -72,6 +72,10 @@ "description": "When `true` indicates the entity has been soft deleted.", "type": "boolean", "default": false + }, + "domain" : { + "description": "Domain the Metrics belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": ["id", "name", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/mlmodel.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/mlmodel.json index 0f0e31a9376..98c7d0bd0c1 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/mlmodel.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/mlmodel.json @@ -264,7 +264,12 @@ "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Domain the MLModel belongs to. When not set, the MLModel inherits the domain from the ML Model Service it belongs to.", + "$ref": "../../type/entityReference.json" } + }, "required": ["id", "name", "algorithm", "service"], "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/pipeline.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/pipeline.json index 20aafa266f3..e755e6b0f04 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/pipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/pipeline.json @@ -253,7 +253,12 @@ "description": "Scheduler Interval for the pipeline in cron format.", "type": "string", "default": null + }, + "domain" : { + "description": "Domain the Pipeline belongs to. When not set, the pipeline inherits the domain from the Pipeline service it belongs to.", + "$ref": "../../type/entityReference.json" } + }, "required": ["id", "name", "service"], "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json index a33d58d01ad..5b900a5f61b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json @@ -1006,6 +1006,14 @@ "sourceUrl": { "description": "Source URL of table.", "$ref": "../../type/basic.json#/definitions/sourceUrl" + }, + "domain" : { + "description": "Domain the table belongs to. When not set, the table inherits the domain from the database schema it belongs to.", + "$ref": "../../type/entityReference.json" + }, + "dataProducts" : { + "description": "List of of data products this entity is part of.", + "$ref" : "../../type/entityReferenceList.json#/definitions/entityReferenceList" } }, "required": [ diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/topic.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/topic.json index cc2768a36e9..1865dd907fa 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/topic.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/topic.json @@ -152,6 +152,14 @@ "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Domain the Topic belongs to. When not set, the Topic inherits the domain from the messaging service it belongs to.", + "$ref": "../../type/entityReference.json" + }, + "dataProducts" : { + "description": "List of of data products this entity is part of.", + "$ref" : "../../type/entityReferenceList.json#/definitions/entityReferenceList" } }, "required": ["id", "name", "partitions", "service"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/domains/dataProduct.json b/openmetadata-spec/src/main/resources/json/schema/entity/domains/dataProduct.json new file mode 100644 index 00000000000..937d48c2d1b --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/domains/dataProduct.json @@ -0,0 +1,66 @@ +{ + "$id": "https://open-metadata.org/schema/entity/domains/dataProduct.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DataProduct", + "description": "A `Data Product` or `Data as a Product` is a logical unit that contains all components to process and store domain data for analytical or data-intensive use cases made available to data consumers.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.domains.DataProduct", + "javaInterfaces": ["org.openmetadata.schema.EntityInterface"], + "properties": { + "id": { + "description": "Unique ID of the Data Product", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "A unique name of the Data Product", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "fullyQualifiedName": { + "description": "FullyQualifiedName is `domain.dataProductName` or `sub-domain.dataProductName`.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "displayName": { + "description": "Name used for display purposes. Example 'Marketing', 'Payments', etc.", + "type": "string" + }, + "description": { + "description": "Description of the Data Product.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/entityVersion" + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "href": { + "description": "Link to the resource corresponding to this entity.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "owner": { + "description": "Owner of this Data Product.", + "$ref": "../../type/entityReference.json" + }, + "experts": { + "description": "List of of users who are experts for this Data Product.", + "$ref": "../../type/entityReferenceList.json#/definitions/entityReferenceList", + "default" : null + }, + "domain": { + "description": "Domain or sub-domain to which this Data Product belongs to.", + "$ref": "../../type/entityReference.json" + }, + "changeDescription": { + "description": "Change that lead to this version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + } + }, + "required": ["id", "name", "description", "domain", "href"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/domains/domain.json b/openmetadata-spec/src/main/resources/json/schema/entity/domains/domain.json index 59848b0f280..bbf774d8c9c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/domains/domain.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/domains/domain.json @@ -80,6 +80,6 @@ "$ref": "../../type/entityHistory.json#/definitions/changeDescription" } }, - "required": ["id", "name", "domainType, ","href"], + "required": ["id", "name", "description", "domainType, ","href"], "additionalProperties": false } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json index daebde5b5e3..606de07db0b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json @@ -178,6 +178,10 @@ "description": "When `true` indicates the entity has been soft deleted.", "type": "boolean", "default": false + }, + "domain" : { + "description": "Domain the Dashboard service belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": ["id", "name", "serviceType"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json index e42c2f255c5..a8430d59e84 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json @@ -332,6 +332,10 @@ "description": "When `true` indicates the entity has been soft deleted.", "type": "boolean", "default": false + }, + "domain" : { + "description": "Domain the Database service belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": ["id", "name", "serviceType"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/messagingService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/messagingService.json index 40414143ad7..174d82ad085 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/messagingService.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/messagingService.json @@ -139,6 +139,10 @@ "description": "When `true` indicates the entity has been soft deleted.", "type": "boolean", "default": false + }, + "domain" : { + "description": "Domain the Messaging service belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": ["id", "name", "serviceType"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/mlmodelService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/mlmodelService.json index b42bb8dbdb0..f115c9faf31 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/mlmodelService.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/mlmodelService.json @@ -131,6 +131,10 @@ "description": "When `true` indicates the entity has been soft deleted.", "type": "boolean", "default": false + }, + "domain" : { + "description": "Domain the MLModel service belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": ["id", "name", "serviceType"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/pipelineService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/pipelineService.json index 2eb80fb12c1..03a05b482ea 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/pipelineService.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/pipelineService.json @@ -180,6 +180,10 @@ "description": "When `true` indicates the entity has been soft deleted.", "type": "boolean", "default": false + }, + "domain" : { + "description": "Domain the Pipeline service belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": [ diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/storageService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/storageService.json index 1b0c9096453..78e9f2d144a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/storageService.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/storageService.json @@ -121,6 +121,10 @@ "description": "When `true` indicates the entity has been soft deleted.", "type": "boolean", "default": false + }, + "domain" : { + "description": "Domain the Storage service belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": ["id", "name", "serviceType"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/teams/team.json b/openmetadata-spec/src/main/resources/json/schema/entity/teams/team.json index 85be6dfec5a..d872b4b79d2 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/teams/team.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/teams/team.json @@ -123,6 +123,10 @@ "policies": { "description": "Policies that is attached to this team.", "$ref": "../../type/entityReferenceList.json#/definitions/entityReferenceList" + }, + "domain" : { + "description": "Domain the Team belongs to.", + "$ref": "../../type/entityReference.json" } }, "required": ["id", "name", "href"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/teams/user.json b/openmetadata-spec/src/main/resources/json/schema/entity/teams/user.json index 74de029b63a..c5f97e998e4 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/teams/user.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/teams/user.json @@ -132,6 +132,10 @@ "isEmailVerified": { "description": "If the User has verified the mail", "type": "boolean" + }, + "domain" : { + "description": "Domain the User belongs to. This is inherited by the team the user belongs to.", + "$ref": "../../type/entityReference.json" } }, "additionalProperties": false,