diff --git a/bootstrap/sql/mysql/v004__create_db_connection_info.sql b/bootstrap/sql/mysql/v004__create_db_connection_info.sql index c3b4c02bad4..db93b6dde74 100644 --- a/bootstrap/sql/mysql/v004__create_db_connection_info.sql +++ b/bootstrap/sql/mysql/v004__create_db_connection_info.sql @@ -15,3 +15,27 @@ CREATE TABLE IF NOT EXISTS thread_entity ( INDEX (updatedAt) ); +CREATE TABLE IF NOT EXISTS glossary_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')), + PRIMARY KEY (id), + UNIQUE KEY unique_name(name), + INDEX (updatedBy), + INDEX (updatedAt) +); +CREATE TABLE IF NOT EXISTS glossary_term_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')), + PRIMARY KEY (id), + UNIQUE KEY unique_name(fullyQualifiedName), + INDEX (updatedBy), + INDEX (updatedAt) +); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java index 0d3fdda73ee..c31ff11fd21 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java @@ -40,9 +40,16 @@ import org.openmetadata.catalog.util.EntityUtil.Fields; @Slf4j public final class Entity { - private static final Map> DAO_MAP = new HashMap<>(); - private static final Map> ENTITY_REPOSITORY_MAP = new HashMap<>(); + // Lower case entity name to canonical entity name map private static final Map CANONICAL_ENTITY_NAME_MAP = new HashMap<>(); + + // Canonical entity name to corresponding EntityDAO map + private static final Map> DAO_MAP = new HashMap<>(); + + // Canonical entity name to corresponding EntityRepository map + private static final Map> ENTITY_REPOSITORY_MAP = new HashMap<>(); + + // Entity class to entity repository map private static final Map, EntityRepository> CLASS_ENTITY_REPOSITORY_MAP = new HashMap<>(); // @@ -70,6 +77,8 @@ public final class Entity { public static final String UNUSED = "unused"; public static final String BOTS = "bots"; public static final String LOCATION = "location"; + public static final String GLOSSARY = "glossary"; + public static final String GLOSSARY_TERM = "glossaryTerm"; // // Policies @@ -92,12 +101,17 @@ public final class Entity { private Entity() {} public static void registerEntity( - Class clazz, String entity, EntityDAO dao, EntityRepository entityRepository) { + Class clazz, String entity, EntityDAO dao, EntityRepository entityRepository) { DAO_MAP.put(entity, dao); ENTITY_REPOSITORY_MAP.put(entity, entityRepository); CANONICAL_ENTITY_NAME_MAP.put(entity.toLowerCase(Locale.ROOT), entity); CLASS_ENTITY_REPOSITORY_MAP.put(clazz, entityRepository); - LOG.info("Registering entity {}", entity); + LOG.info( + "Registering entity {} {} {} {}", + clazz, + entity, + dao.getEntityClass().getSimpleName(), + entityRepository.getClass().getSimpleName()); } public static EntityReference getEntityReference(String entity, UUID id) throws IOException { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java index 7d6d2eb4a5c..95482d3c959 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Collectors; +import javax.ws.rs.core.Response; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; @@ -31,6 +32,7 @@ import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.common.xcontent.XContentType; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.entity.data.Dashboard; +import org.openmetadata.catalog.entity.data.Glossary; import org.openmetadata.catalog.entity.data.Pipeline; import org.openmetadata.catalog.entity.data.Table; import org.openmetadata.catalog.entity.data.Topic; @@ -67,7 +69,8 @@ public class ElasticSearchIndexDefinition { DASHBOARD_SEARCH_INDEX("dashboard_search_index", "/elasticsearch/dashboard_index_mapping.json"), PIPELINE_SEARCH_INDEX("pipeline_search_index", "/elasticsearch/pipeline_index_mapping.json"), USER_SEARCH_INDEX("user_search_index", "/elasticsearch/user_index_mapping.json"), - TEAM_SEARCH_INDEX("team_search_index", "/elasticsearch/team_index_mapping.json"); + TEAM_SEARCH_INDEX("team_search_index", "/elasticsearch/team_index_mapping.json"), + GLOSSARY_SEARCH_INDEX("glossary_search_index", "/elasticsearch/glossary_index_mapping.json"); public final String indexName; public final String indexMappingFile; @@ -197,6 +200,8 @@ public class ElasticSearchIndexDefinition { return ElasticSearchIndexType.USER_SEARCH_INDEX; } else if (type.equalsIgnoreCase(Entity.TEAM)) { return ElasticSearchIndexType.TEAM_SEARCH_INDEX; + } else if (type.equalsIgnoreCase(Entity.GLOSSARY)) { + return ElasticSearchIndexType.GLOSSARY_SEARCH_INDEX; } throw new RuntimeException("Failed to find index doc for type " + type); } @@ -852,3 +857,61 @@ class TeamESIndex { return teamESIndexBuilder; } } + +@EqualsAndHashCode(callSuper = true) +@Getter +@SuperBuilder(builderMethodName = "internalBuilder") +@Value +@JsonInclude(JsonInclude.Include.NON_NULL) +class GlossaryESIndex extends ElasticSearchIndex { + @JsonProperty("glossary_id") + String glossaryId; + + public static GlossaryESIndexBuilder builder(Glossary glossary, int responseCode) { + List tags = new ArrayList<>(); + List taskNames = new ArrayList<>(); + List taskDescriptions = new ArrayList<>(); + List suggest = new ArrayList<>(); + suggest.add(ElasticSearchSuggest.builder().input(glossary.getName()).weight(5).build()); + suggest.add(ElasticSearchSuggest.builder().input(glossary.getDisplayName()).weight(10).build()); + + if (glossary.getTags() != null) { + glossary.getTags().forEach(tag -> tags.add(tag.getTagFQN())); + } + + Long updatedTimestamp = glossary.getUpdatedAt(); + ParseTags parseTags = new ParseTags(tags); + String description = glossary.getDescription() != null ? glossary.getDescription() : ""; + String displayName = glossary.getDisplayName() != null ? glossary.getDisplayName() : ""; + GlossaryESIndexBuilder builder = + internalBuilder() + .glossaryId(glossary.getId().toString()) + .name(glossary.getDisplayName()) + .displayName(description) + .description(displayName) + .fqdn(glossary.getName()) + .lastUpdatedTimestamp(updatedTimestamp) + .entityType("glossary") + .suggest(suggest) + .tags(parseTags.tags) + .tier(parseTags.tierTag); + + if (glossary.getOwner() != null) { + builder.owner(glossary.getOwner().getId().toString()); + } + + ESChangeDescription esChangeDescription = null; + if (glossary.getChangeDescription() != null) { + esChangeDescription = + ESChangeDescription.builder().updatedAt(updatedTimestamp).updatedBy(glossary.getUpdatedBy()).build(); + esChangeDescription.setFieldsAdded(glossary.getChangeDescription().getFieldsAdded()); + esChangeDescription.setFieldsDeleted(glossary.getChangeDescription().getFieldsDeleted()); + esChangeDescription.setFieldsUpdated(glossary.getChangeDescription().getFieldsUpdated()); + } else if (responseCode == Response.Status.CREATED.getStatusCode()) { + esChangeDescription = + ESChangeDescription.builder().updatedAt(updatedTimestamp).updatedBy(glossary.getUpdatedBy()).build(); + } + builder.changeDescriptions(esChangeDescription != null ? List.of(esChangeDescription) : null); + return builder; + } +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java index 34fa5f571b0..4b33628f522 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java @@ -20,6 +20,7 @@ import java.sql.SQLException; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.UUID; import org.jdbi.v3.core.mapper.RowMapper; import org.jdbi.v3.core.statement.StatementContext; import org.jdbi.v3.sqlobject.CreateSqlObject; @@ -33,6 +34,8 @@ import org.openmetadata.catalog.entity.Bots; import org.openmetadata.catalog.entity.data.Chart; import org.openmetadata.catalog.entity.data.Dashboard; import org.openmetadata.catalog.entity.data.Database; +import org.openmetadata.catalog.entity.data.Glossary; +import org.openmetadata.catalog.entity.data.GlossaryTerm; import org.openmetadata.catalog.entity.data.Location; import org.openmetadata.catalog.entity.data.Metrics; import org.openmetadata.catalog.entity.data.MlModel; @@ -58,6 +61,8 @@ import org.openmetadata.catalog.jdbi3.DashboardRepository.DashboardEntityInterfa import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface; import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface; import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface; +import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface; +import org.openmetadata.catalog.jdbi3.GlossaryTermRepository.GlossaryTermEntityInterface; import org.openmetadata.catalog.jdbi3.LocationRepository.LocationEntityInterface; import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface; import org.openmetadata.catalog.jdbi3.MetricsRepository.MetricsEntityInterface; @@ -135,6 +140,12 @@ public interface CollectionDAO { @CreateSqlObject MlModelDAO mlModelDAO(); + @CreateSqlObject + GlossaryDAO glossaryDAO(); + + @CreateSqlObject + GlossaryTermDAO glossaryTermDAO(); + @CreateSqlObject BotsDAO botsDAO(); @@ -331,6 +342,10 @@ public interface CollectionDAO { } interface EntityRelationshipDAO { + default int insert(UUID fromId, UUID toId, String fromEntity, String toEntity, int relation) { + return insert(fromId.toString(), toId.toString(), fromEntity, toEntity, relation); + } + @SqlUpdate( "INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) " + "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation)") @@ -690,6 +705,50 @@ public interface CollectionDAO { } } + interface GlossaryDAO extends EntityDAO { + @Override + default String getTableName() { + return "glossary_entity"; + } + + @Override + default Class getEntityClass() { + return Glossary.class; + } + + @Override + default String getNameColumn() { + return "name"; + } + + @Override + default EntityReference getEntityReference(Glossary entity) { + return new GlossaryEntityInterface(entity).getEntityReference(); + } + } + + interface GlossaryTermDAO extends EntityDAO { + @Override + default String getTableName() { + return "glossary_term_entity"; + } + + @Override + default Class getEntityClass() { + return GlossaryTerm.class; + } + + @Override + default String getNameColumn() { + return "fullyQualifiedName"; + } + + @Override + default EntityReference getEntityReference(GlossaryTerm entity) { + return new GlossaryTermEntityInterface(entity).getEntityReference(); + } + } + interface AirflowPipelineDAO extends EntityDAO { @Override default String getTableName() { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java index f9eebf58951..b28b0d32f45 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java @@ -30,7 +30,6 @@ import java.net.URI; import java.security.GeneralSecurityException; import java.text.ParseException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -76,7 +75,6 @@ import org.openmetadata.catalog.util.ResultList; import org.openmetadata.common.utils.CipherText; import org.openmetadata.common.utils.CommonUtil; -@Slf4j /** * This is the base class used by Entity Resources to perform READ and WRITE operations to the backend database to * Create, Retrieve, Update, and Delete entities. @@ -108,6 +106,7 @@ import org.openmetadata.common.utils.CommonUtil; * relationship table when required to ensure, the data stored is efficiently and consistently, and relationship * information does not become stale. */ +@Slf4j public abstract class EntityRepository { private final String collectionPath; private final Class entityClass; @@ -561,21 +560,18 @@ public abstract class EntityRepository { } } + public static final Fields FIELDS_OWNER = new Fields(List.of("owner"), "owner"); + public final EntityReference getOriginalOwner(T entity) throws IOException, ParseException { - final String FIELDS = "owner"; - final List FIELD_LIST = Arrays.asList(FIELDS.replace(" ", "").split(",")); - EntityUtil.Fields fields = new EntityUtil.Fields(FIELD_LIST, FIELDS); - EntityReference owner = null; // Try to find the owner if entity exists try { String fqn = getFullyQualifiedName(entity); - entity = getByName(null, fqn, fields); - owner = helper(entity).validateOwnerOrNull(); + entity = getByName(null, fqn, FIELDS_OWNER); + return helper(entity).validateOwnerOrNull(); } catch (EntityNotFoundException e) { - // If entity is not found, we can return null for owner and ignore - // this exception + // If entity is not found, we can return null for owner and ignore this exception } - return owner; + return null; } protected EntityReference getOwner(T entity) throws IOException, ParseException { @@ -601,7 +597,7 @@ public abstract class EntityRepository { } protected List getTags(String fqn) { - return !supportsOwner ? null : daoCollection.tagDAO().getTags(fqn); + return !supportsTags ? null : daoCollection.tagDAO().getTags(fqn); } protected List getFollowers(T entity) throws IOException { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java new file mode 100644 index 00000000000..4cfa70f52ad --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.jdbi3; + +import static org.openmetadata.catalog.Entity.helper; + +import java.io.IOException; +import java.net.URI; +import java.text.ParseException; +import java.util.List; +import java.util.UUID; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.entity.data.Glossary; +import org.openmetadata.catalog.resources.glossary.GlossaryResource; +import org.openmetadata.catalog.type.ChangeDescription; +import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.type.TagLabel; +import org.openmetadata.catalog.util.EntityInterface; +import org.openmetadata.catalog.util.EntityUtil; +import org.openmetadata.catalog.util.EntityUtil.Fields; +import org.openmetadata.catalog.util.JsonUtils; + +public class GlossaryRepository extends EntityRepository { + private static final Fields GLOSSARY_UPDATE_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "owner,tags"); + private static final Fields GLOSSARY_PATCH_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "owner,tags"); + private final CollectionDAO dao; + + public GlossaryRepository(CollectionDAO dao) { + super( + GlossaryResource.COLLECTION_PATH, + Entity.GLOSSARY, + Glossary.class, + dao.glossaryDAO(), + dao, + GLOSSARY_PATCH_FIELDS, + GLOSSARY_UPDATE_FIELDS, + true, + true, + false); + this.dao = dao; + } + + @Transaction + public EntityReference getOwnerReference(Glossary glossary) throws IOException { + return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), glossary.getOwner()); + } + + @Override + public Glossary setFields(Glossary glossary, Fields fields) throws IOException, ParseException { + glossary.setOwner(fields.contains("owner") ? getOwner(glossary) : null); + glossary.setTags(fields.contains("tags") ? getTags(glossary.getName()) : null); + return glossary; + } + + @Override + public void prepare(Glossary glossary) throws IOException, ParseException { + glossary.setOwner(helper(glossary).validateOwnerOrNull()); + // TODO validate reviewers + glossary.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), glossary.getTags())); + } + + @Override + public void storeEntity(Glossary glossary, boolean update) throws IOException { + // Relationships and fields such as href are derived and not stored as part of json + EntityReference owner = glossary.getOwner(); + List tags = glossary.getTags(); + // TODO Add relationships for reviewers + + // Don't store owner, href and tags as JSON. Build it on the fly based on relationships + glossary.withOwner(null).withHref(null).withTags(null); + + if (update) { + dao.glossaryDAO().update(glossary.getId(), JsonUtils.pojoToJson(glossary)); + } else { + dao.glossaryDAO().insert(glossary); + } + + // Restore the relationships + glossary.withOwner(owner).withTags(tags); + } + + @Override + public void storeRelationships(Glossary glossary) { + // TODO Add relationships for related terms, and reviewers + setOwner(glossary, glossary.getOwner()); + applyTags(glossary); + } + + @Override + public void restorePatchAttributes(Glossary original, Glossary updated) {} + + @Override + public EntityInterface getEntityInterface(Glossary entity) { + return new GlossaryEntityInterface(entity); + } + + @Override + public EntityUpdater getUpdater(Glossary original, Glossary updated, Operation operation) { + return new GlossaryUpdater(original, updated, operation); + } + + public static class GlossaryEntityInterface implements EntityInterface { + private final Glossary entity; + + public GlossaryEntityInterface(Glossary entity) { + this.entity = entity; + } + + @Override + public UUID getId() { + return entity.getId(); + } + + @Override + public String getDescription() { + return entity.getDescription(); + } + + @Override + public String getDisplayName() { + return entity.getDisplayName(); + } + + @Override + public Boolean isDeleted() { + return entity.getDeleted(); + } + + @Override + public EntityReference getOwner() { + return entity.getOwner(); + } + + @Override + public String getFullyQualifiedName() { + return entity.getName(); + } + + @Override + public List getTags() { + return entity.getTags(); + } + + @Override + public Double getVersion() { + return entity.getVersion(); + } + + @Override + public String getUpdatedBy() { + return entity.getUpdatedBy(); + } + + @Override + public long getUpdatedAt() { + return entity.getUpdatedAt(); + } + + @Override + public URI getHref() { + return entity.getHref(); + } + + @Override + public ChangeDescription getChangeDescription() { + return entity.getChangeDescription(); + } + + @Override + public EntityReference getEntityReference() { + return new EntityReference() + .withId(getId()) + .withName(getFullyQualifiedName()) + .withDescription(getDescription()) + .withDisplayName(getDisplayName()) + .withType(Entity.GLOSSARY); + } + + @Override + public Glossary getEntity() { + return entity; + } + + @Override + public EntityReference getContainer() { + return null; + } + + @Override + public void setId(UUID id) { + entity.setId(id); + } + + @Override + public void setDescription(String description) { + entity.setDescription(description); + } + + @Override + public void setDisplayName(String displayName) { + entity.setDisplayName(displayName); + } + + @Override + public void setUpdateDetails(String updatedBy, long updatedAt) { + entity.setUpdatedBy(updatedBy); + entity.setUpdatedAt(updatedAt); + } + + @Override + public void setChangeDescription(Double newVersion, ChangeDescription changeDescription) { + entity.setVersion(newVersion); + entity.setChangeDescription(changeDescription); + } + + @Override + public void setOwner(EntityReference owner) { + entity.setOwner(owner); + } + + @Override + public void setDeleted(boolean flag) { + entity.setDeleted(flag); + } + + @Override + public Glossary withHref(URI href) { + return entity.withHref(href); + } + + @Override + public void setTags(List tags) { + entity.setTags(tags); + } + } + + /** Handles entity updated from PUT and POST operation. */ + public class GlossaryUpdater extends EntityUpdater { + public GlossaryUpdater(Glossary original, Glossary updated, Operation operation) { + super(original, updated, operation); + } + } +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java new file mode 100644 index 00000000000..ebd30b68ce0 --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.jdbi3; + +import java.io.IOException; +import java.net.URI; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.entity.data.GlossaryTerm; +import org.openmetadata.catalog.resources.glossary.GlossaryResource; +import org.openmetadata.catalog.resources.glossary.GlossaryTermResource; +import org.openmetadata.catalog.type.ChangeDescription; +import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.type.Relationship; +import org.openmetadata.catalog.type.TagLabel; +import org.openmetadata.catalog.util.EntityInterface; +import org.openmetadata.catalog.util.EntityUtil; +import org.openmetadata.catalog.util.EntityUtil.Fields; +import org.openmetadata.catalog.util.JsonUtils; + +@Slf4j +public class GlossaryTermRepository extends EntityRepository { + private static final Fields UPDATE_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "tags"); + private static final Fields PATCH_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "tags"); + private final CollectionDAO dao; + + public GlossaryTermRepository(CollectionDAO dao) { + super( + GlossaryTermResource.COLLECTION_PATH, + Entity.GLOSSARY_TERM, + GlossaryTerm.class, + dao.glossaryTermDAO(), + dao, + PATCH_FIELDS, + UPDATE_FIELDS, + true, + false, + false); + this.dao = dao; + } + + @Override + public GlossaryTerm setFields(GlossaryTerm entity, Fields fields) throws IOException, ParseException { + entity.setGlossary(getGlossary(entity)); + entity.setTags(fields.contains("tags") ? getTags(entity.getFullyQualifiedName()) : null); + return entity; + } + + @Override + public void prepare(GlossaryTerm entity) throws IOException, ParseException { + // Set fully qualified name + if (entity.getParent() == null) { + entity.setFullyQualifiedName(entity.getGlossary().getName() + "." + entity.getName()); + } else { + entity.setFullyQualifiedName(entity.getParent().getName() + "." + entity.getName()); + } + + // Validate related terms + List validatedRelatedTerms = new ArrayList<>(); + for (EntityReference related : entity.getRelatedTerms()) { + validatedRelatedTerms.add(daoCollection.glossaryTermDAO().findEntityReferenceById(related.getId())); + } + entity.setRelatedTerms(validatedRelatedTerms); + + // Set tags + entity.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), entity.getTags())); + } + + @Override + public void storeEntity(GlossaryTerm entity, boolean update) throws IOException { + // Relationships and fields such as href are derived and not stored as part of json + List tags = entity.getTags(); + // TODO Add relationships for reviewers + EntityReference glossary = entity.getGlossary(); + EntityReference parentTerm = entity.getParent(); + + // Don't store owner, dashboard, href and tags as JSON. Build it on the fly based on relationships + entity.withGlossary(null).withParent(null).withHref(null).withTags(null); + + if (update) { + dao.glossaryTermDAO().update(entity.getId(), JsonUtils.pojoToJson(entity)); + } else { + dao.glossaryTermDAO().insert(entity); + } + + // Restore the relationships + entity.withGlossary(glossary).withParent(parentTerm).withTags(tags); + } + + @Override + public void storeRelationships(GlossaryTerm entity) { + // TODO Add relationships for related terms, and reviewers + daoCollection + .relationshipDAO() + .insert( + entity.getGlossary().getId(), + entity.getId(), + Entity.GLOSSARY, + Entity.GLOSSARY_TERM, + Relationship.CONTAINS.ordinal()); + if (entity.getParent() != null) { + daoCollection + .relationshipDAO() + .insert( + entity.getParent().getId(), + entity.getId(), + Entity.GLOSSARY, + Entity.GLOSSARY_TERM, + Relationship.CONTAINS.ordinal()); + } + + applyTags(entity); + } + + @Override + public void restorePatchAttributes(GlossaryTerm original, GlossaryTerm updated) {} + + protected EntityReference getGlossary(GlossaryTerm term) throws IOException { + List refs = + daoCollection + .relationshipDAO() + .findFrom( + term.getId().toString(), Entity.GLOSSARY_TERM, Relationship.CONTAINS.ordinal(), Entity.GLOSSARY, null); + if (refs.size() != 1) { + LOG.warn( + "Possible database issues - multiple owners found for entity {} with type {}", term.getId(), Entity.GLOSSARY); + } + return daoCollection.glossaryDAO().findEntityReferenceById(UUID.fromString(refs.get(0))); + } + + @Override + public EntityInterface getEntityInterface(GlossaryTerm entity) { + return new GlossaryTermEntityInterface(entity); + } + + @Override + public EntityUpdater getUpdater(GlossaryTerm original, GlossaryTerm updated, Operation operation) { + return new GlossaryTermUpdater(original, updated, operation); + } + + public static class GlossaryTermEntityInterface implements EntityInterface { + private final GlossaryTerm entity; + + public GlossaryTermEntityInterface(GlossaryTerm entity) { + this.entity = entity; + } + + @Override + public UUID getId() { + return entity.getId(); + } + + @Override + public String getDescription() { + return entity.getDescription(); + } + + @Override + public String getDisplayName() { + return entity.getDisplayName(); + } + + @Override + public Boolean isDeleted() { + return entity.getDeleted(); + } + + @Override + public EntityReference getOwner() { + return null; + } + + @Override + public String getFullyQualifiedName() { + return entity.getFullyQualifiedName(); + } + + @Override + public List getTags() { + return entity.getTags(); + } + + @Override + public Double getVersion() { + return entity.getVersion(); + } + + @Override + public String getUpdatedBy() { + return entity.getUpdatedBy(); + } + + @Override + public long getUpdatedAt() { + return entity.getUpdatedAt(); + } + + @Override + public URI getHref() { + return entity.getHref(); + } + + @Override + public ChangeDescription getChangeDescription() { + return entity.getChangeDescription(); + } + + @Override + public EntityReference getEntityReference() { + return new EntityReference() + .withId(getId()) + .withName(getFullyQualifiedName()) + .withDescription(getDescription()) + .withDisplayName(getDisplayName()) + .withType(Entity.GLOSSARY_TERM); + } + + @Override + public GlossaryTerm getEntity() { + return entity; + } + + @Override + public EntityReference getContainer() { + return null; + } + + @Override + public void setId(UUID id) { + entity.setId(id); + } + + @Override + public void setDescription(String description) { + entity.setDescription(description); + } + + @Override + public void setDisplayName(String displayName) { + entity.setDisplayName(displayName); + } + + @Override + public void setUpdateDetails(String updatedBy, long updatedAt) { + entity.setUpdatedBy(updatedBy); + entity.setUpdatedAt(updatedAt); + } + + @Override + public void setChangeDescription(Double newVersion, ChangeDescription changeDescription) { + entity.setVersion(newVersion); + entity.setChangeDescription(changeDescription); + } + + @Override + public void setDeleted(boolean flag) { + entity.setDeleted(flag); + } + + @Override + public GlossaryTerm withHref(URI href) { + return entity.withHref(href); + } + + @Override + public void setTags(List tags) { + entity.setTags(tags); + } + } + + /** Handles entity updated from PUT and POST operation. */ + public class GlossaryTermUpdater extends EntityUpdater { + public GlossaryTermUpdater(GlossaryTerm original, GlossaryTerm updated, Operation operation) { + super(original, updated, operation); + } + + @Override + public void entitySpecificUpdate() throws IOException { + // TODO + } + } +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java new file mode 100644 index 00000000000..9a1284f3d32 --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java @@ -0,0 +1,397 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.glossary; + +import com.google.inject.Inject; +import io.swagger.annotations.Api; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.security.GeneralSecurityException; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import javax.json.JsonPatch; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.PATCH; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.api.data.CreateGlossary; +import org.openmetadata.catalog.entity.data.Glossary; +import org.openmetadata.catalog.jdbi3.CollectionDAO; +import org.openmetadata.catalog.jdbi3.GlossaryRepository; +import org.openmetadata.catalog.resources.Collection; +import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.security.SecurityUtil; +import org.openmetadata.catalog.type.EntityHistory; +import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.type.Include; +import org.openmetadata.catalog.util.EntityUtil.Fields; +import org.openmetadata.catalog.util.RestUtil; +import org.openmetadata.catalog.util.RestUtil.DeleteResponse; +import org.openmetadata.catalog.util.RestUtil.PatchResponse; +import org.openmetadata.catalog.util.RestUtil.PutResponse; +import org.openmetadata.catalog.util.ResultList; + +@Path("/v1/glossaries") +@Api(value = "Glossary collection", tags = "Glossary collection") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "glossaries") +public class GlossaryResource { + public static final String COLLECTION_PATH = "v1/glossaries/"; + private final GlossaryRepository dao; + private final Authorizer authorizer; + + public static void addHref(UriInfo uriInfo, EntityReference ref) { + ref.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, ref.getId())); + } + + public static List addHref(UriInfo uriInfo, List glossaries) { + Optional.ofNullable(glossaries).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i)); + return glossaries; + } + + public static Glossary addHref(UriInfo uriInfo, Glossary glossary) { + glossary.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, glossary.getId())); + Entity.withHref(uriInfo, glossary.getOwner()); + return glossary; + } + + @Inject + public GlossaryResource(CollectionDAO dao, Authorizer authorizer) { + Objects.requireNonNull(dao, "GlossaryRepository must not be null"); + this.dao = new GlossaryRepository(dao); + this.authorizer = authorizer; + } + + public static class GlossaryList extends ResultList { + @SuppressWarnings("unused") + GlossaryList() { + // Empty constructor needed for deserialization + } + + public GlossaryList(List data, String beforeCursor, String afterCursor, int total) + throws GeneralSecurityException, UnsupportedEncodingException { + super(data, beforeCursor, afterCursor, total); + } + } + + static final String FIELDS = "owner,tags"; + public static final List FIELD_LIST = Arrays.asList(FIELDS.replaceAll(" ", "").split(",")); + + @GET + @Valid + @Operation( + summary = "List Glossaries", + tags = "glossaries", + description = + "Get a list of glossaries. Use `fields` parameter to get only necessary fields. " + + " Use cursor-based pagination to limit the number " + + "entries in the list using `limit` and `before` or `after` query params.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of glossaries", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter(description = "Limit the number glossaries returned. (1 to 1000000, " + "default = 10)") + @DefaultValue("10") + @Min(1) + @Max(1000000) + @QueryParam("limit") + int limitParam, + @Parameter(description = "Returns list of glossaries before this cursor", schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter(description = "Returns list of glossaries after this cursor", schema = @Schema(type = "string")) + @QueryParam("after") + String after, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException, GeneralSecurityException, ParseException { + RestUtil.validateCursors(before, after); + Fields fields = new Fields(FIELD_LIST, fieldsParam); + + ResultList glossary; + if (before != null) { // Reverse paging + glossary = dao.listBefore(uriInfo, fields, null, limitParam, before, include); // Ask for one extra entry + } else { // Forward paging or first page + glossary = dao.listAfter(uriInfo, fields, null, limitParam, after, include); + } + addHref(uriInfo, glossary.getData()); + return glossary; + } + + @GET + @Path("/{id}") + @Operation( + summary = "Get a glossary", + tags = "glossaries", + description = "Get a glossary by `id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The glossary", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))), + @ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found") + }) + public Glossary get( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") String id, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException, ParseException { + Fields fields = new Fields(FIELD_LIST, fieldsParam); + return addHref(uriInfo, dao.get(uriInfo, id, fields, include)); + } + + @GET + @Path("/name/{name}") + @Operation( + summary = "Get a glossary by name", + tags = "glossaries", + description = "Get a glossary by name.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The glossary", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))), + @ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found") + }) + public Glossary getByName( + @Context UriInfo uriInfo, + @PathParam("name") String name, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException, ParseException { + Fields fields = new Fields(FIELD_LIST, fieldsParam); + Glossary glossary = dao.getByName(uriInfo, name, fields, include); + return addHref(uriInfo, glossary); + } + + @GET + @Path("/{id}/versions") + @Operation( + summary = "List glossary versions", + tags = "glossaries", + description = "Get a list of all the versions of a glossary identified by `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of glossary versions", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class))) + }) + public EntityHistory listVersions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id) + throws IOException, ParseException { + return dao.listVersions(id); + } + + @GET + @Path("/{id}/versions/{version}") + @Operation( + summary = "Get a version of the glossaries", + tags = "glossaries", + description = "Get a version of the glossary by given `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "glossaries", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))), + @ApiResponse( + responseCode = "404", + description = "Glossary for instance {id} and version {version} is " + "not found") + }) + public Glossary getVersion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id, + @Parameter( + description = "glossary version number in the form `major`.`minor`", + schema = @Schema(type = "string", example = "0.1 or 1.1")) + @PathParam("version") + String version) + throws IOException, ParseException { + return dao.getVersion(id, version); + } + + @POST + @Operation( + summary = "Create a glossary", + tags = "glossaries", + description = "Create a new glossary.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The glossary", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = CreateGlossary.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response create( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create) + throws IOException, ParseException { + SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + Glossary glossary = getGlossary(securityContext, create); + glossary = addHref(uriInfo, dao.create(uriInfo, glossary)); + return Response.created(glossary.getHref()).entity(glossary).build(); + } + + @PATCH + @Path("/{id}") + @Operation( + summary = "Update a glossary", + tags = "glossaries", + description = "Update an existing glossary using JsonPatch.", + externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902")) + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + public Response updateDescription( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") String id, + @RequestBody( + description = "JsonPatch with array of operations", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON_PATCH_JSON, + examples = { + @ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]") + })) + JsonPatch patch) + throws IOException, ParseException { + Fields fields = new Fields(FIELD_LIST, FIELDS); + Glossary glossary = dao.get(uriInfo, id, fields); + SecurityUtil.checkAdminRoleOrPermissions( + authorizer, securityContext, dao.getEntityInterface(glossary).getEntityReference(), patch); + PatchResponse response = + dao.patch(uriInfo, UUID.fromString(id), securityContext.getUserPrincipal().getName(), patch); + addHref(uriInfo, response.getEntity()); + return response.toResponse(); + } + + @PUT + @Operation( + summary = "Create or update a glossary", + tags = "glossaries", + description = "Create a new glossary, if it does not exist or update an existing glossary.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The glossary", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response createOrUpdate( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create) + throws IOException, ParseException { + Glossary glossary = getGlossary(securityContext, create); + EntityReference owner = dao.getOriginalOwner(glossary); + SecurityUtil.checkAdminRoleOrPermissions(authorizer, securityContext, owner); + PutResponse response = dao.createOrUpdate(uriInfo, glossary); + addHref(uriInfo, response.getEntity()); + return response.toResponse(); + } + + @DELETE + @Path("/{id}") + @Operation( + summary = "Delete a Glossary", + tags = "glossaries", + description = "Delete a glossary by `id`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "glossary for instance {id} is not found") + }) + public Response delete(@Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("id") String id) + throws IOException, ParseException { + SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + DeleteResponse response = dao.delete(securityContext.getUserPrincipal().getName(), id); + return response.toResponse(); + } + + private Glossary getGlossary(SecurityContext securityContext, CreateGlossary create) { + return new Glossary() + .withId(UUID.randomUUID()) + .withName(create.getName()) + .withDisplayName(create.getDisplayName()) + .withDescription(create.getDescription()) + .withReviewers(create.getReviewers()) + .withTags(create.getTags()) + .withOwner(create.getOwner()) + .withUpdatedBy(securityContext.getUserPrincipal().getName()) + .withUpdatedAt(System.currentTimeMillis()); + } +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResource.java new file mode 100644 index 00000000000..07e6c4b03ec --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResource.java @@ -0,0 +1,401 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.glossary; + +import com.google.inject.Inject; +import io.swagger.annotations.Api; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.security.GeneralSecurityException; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import javax.json.JsonPatch; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.PATCH; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.api.data.CreateGlossaryTerm; +import org.openmetadata.catalog.entity.data.Glossary; +import org.openmetadata.catalog.entity.data.GlossaryTerm; +import org.openmetadata.catalog.jdbi3.CollectionDAO; +import org.openmetadata.catalog.jdbi3.GlossaryTermRepository; +import org.openmetadata.catalog.resources.Collection; +import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.security.SecurityUtil; +import org.openmetadata.catalog.type.EntityHistory; +import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.type.Include; +import org.openmetadata.catalog.util.EntityUtil.Fields; +import org.openmetadata.catalog.util.RestUtil; +import org.openmetadata.catalog.util.RestUtil.DeleteResponse; +import org.openmetadata.catalog.util.RestUtil.PatchResponse; +import org.openmetadata.catalog.util.RestUtil.PutResponse; +import org.openmetadata.catalog.util.ResultList; + +@Path("/v1/glossaryTerms") +@Api(value = "Glossary collection", tags = "Glossary collection") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "glossaries") +public class GlossaryTermResource { + public static final String COLLECTION_PATH = "v1/glossaryTerms/"; + private final GlossaryTermRepository dao; + private final Authorizer authorizer; + + public static List addHref(UriInfo uriInfo, List terms) { + Optional.ofNullable(terms).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i)); + return terms; + } + + public static GlossaryTerm addHref(UriInfo uriInfo, GlossaryTerm term) { + term.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, term.getId())); + Entity.withHref(uriInfo, term.getGlossary()); + return term; + } + + @Inject + public GlossaryTermResource(CollectionDAO dao, Authorizer authorizer) { + Objects.requireNonNull(dao, "GlossaryTermRepository must not be null"); + this.dao = new GlossaryTermRepository(dao); + this.authorizer = authorizer; + } + + public static class GlossaryTermList extends ResultList { + @SuppressWarnings("unused") + GlossaryTermList() { + // Empty constructor needed for deserialization + } + + public GlossaryTermList(List data, String beforeCursor, String afterCursor, int total) + throws GeneralSecurityException, UnsupportedEncodingException { + super(data, beforeCursor, afterCursor, total); + } + } + + static final String FIELDS = "tags"; + public static final List FIELD_LIST = Arrays.asList(FIELDS.replaceAll(" ", "").split(",")); + + @GET + @Valid + @Operation( + summary = "List glossary terms", + tags = "glossaries", + description = + "Get a list of glossary terms. Use `fields` parameter to get only necessary fields. " + + " Use cursor-based pagination to limit the number " + + "entries in the list using `limit` and `before` or `after` query params.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of glossary terms", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryTermList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter(description = "Limit the number glossary terms returned. (1 to 1000000, " + "default = 10)") + @DefaultValue("10") + @Min(1) + @Max(1000000) + @QueryParam("limit") + int limitParam, + @Parameter(description = "Returns list of glossary terms before this cursor", schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter(description = "Returns list of glossary terms after this cursor", schema = @Schema(type = "string")) + @QueryParam("after") + String after, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException, GeneralSecurityException, ParseException { + RestUtil.validateCursors(before, after); + Fields fields = new Fields(FIELD_LIST, fieldsParam); + + ResultList terms; + if (before != null) { // Reverse paging + terms = dao.listBefore(uriInfo, fields, null, limitParam, before, include); // Ask for one extra entry + } else { // Forward paging or first page + terms = dao.listAfter(uriInfo, fields, null, limitParam, after, include); + } + addHref(uriInfo, terms.getData()); + return terms; + } + + @GET + @Path("/{id}") + @Operation( + summary = "Get a glossary", + tags = "glossaries", + description = "Get a glossary by `id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The glossary", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))), + @ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found") + }) + public GlossaryTerm get( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") String id, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException, ParseException { + Fields fields = new Fields(FIELD_LIST, fieldsParam); + return addHref(uriInfo, dao.get(uriInfo, id, fields, include)); + } + + @GET + @Path("/name/{name}") + @Operation( + summary = "Get a glossary by name", + tags = "glossaries", + description = "Get a glossary by name.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The glossary", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))), + @ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found") + }) + public GlossaryTerm getByName( + @Context UriInfo uriInfo, + @PathParam("name") String name, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException, ParseException { + Fields fields = new Fields(FIELD_LIST, fieldsParam); + GlossaryTerm term = dao.getByName(uriInfo, name, fields, include); + return addHref(uriInfo, term); + } + + @GET + @Path("/{id}/versions") + @Operation( + summary = "List glossary versions", + tags = "glossaries", + description = "Get a list of all the versions of a glossary identified by `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of glossary versions", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class))) + }) + public EntityHistory listVersions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id) + throws IOException, ParseException { + return dao.listVersions(id); + } + + @GET + @Path("/{id}/versions/{version}") + @Operation( + summary = "Get a version of the glossaries", + tags = "glossaries", + description = "Get a version of the glossary by given `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "glossaries", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))), + @ApiResponse( + responseCode = "404", + description = "Glossary for instance {id} and version {version} is " + "not found") + }) + public GlossaryTerm getVersion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id, + @Parameter( + description = "glossary version number in the form `major`.`minor`", + schema = @Schema(type = "string", example = "0.1 or 1.1")) + @PathParam("version") + String version) + throws IOException, ParseException { + return dao.getVersion(id, version); + } + + @POST + @Operation( + summary = "Create a glossary", + tags = "glossaries", + description = "Create a new glossary.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The glossary", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryTerm.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response create( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossaryTerm create) + throws IOException, ParseException { + SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + GlossaryTerm term = getGlossaryTerm(securityContext, create); + term = addHref(uriInfo, dao.create(uriInfo, term)); + return Response.created(term.getHref()).entity(term).build(); + } + + @PATCH + @Path("/{id}") + @Operation( + summary = "Update a glossary", + tags = "glossaries", + description = "Update an existing glossary using JsonPatch.", + externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902")) + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + public Response updateDescription( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") String id, + @RequestBody( + description = "JsonPatch with array of operations", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON_PATCH_JSON, + examples = { + @ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]") + })) + JsonPatch patch) + throws IOException, ParseException { + Fields fields = new Fields(FIELD_LIST, FIELDS); + GlossaryTerm term = dao.get(uriInfo, id, fields); + SecurityUtil.checkAdminRoleOrPermissions( + authorizer, securityContext, dao.getEntityInterface(term).getEntityReference(), patch); + PatchResponse response = + dao.patch(uriInfo, UUID.fromString(id), securityContext.getUserPrincipal().getName(), patch); + addHref(uriInfo, response.getEntity()); + return response.toResponse(); + } + + @PUT + @Operation( + summary = "Create or update a glossary", + tags = "glossaries", + description = "Create a new glossary, if it does not exist or update an existing glossary.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The glossary", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryTerm.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response createOrUpdate( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossaryTerm create) + throws IOException, ParseException { + GlossaryTerm term = getGlossaryTerm(securityContext, create); + EntityReference owner = dao.getOriginalOwner(term); + SecurityUtil.checkAdminRoleOrPermissions(authorizer, securityContext, owner); + PutResponse response = dao.createOrUpdate(uriInfo, term); + addHref(uriInfo, response.getEntity()); + return response.toResponse(); + } + + @DELETE + @Path("/{id}") + @Operation( + summary = "Delete a Glossary", + tags = "glossaries", + description = "Delete a glossary by `id`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "glossary for instance {id} is not found") + }) + public Response delete(@Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("id") String id) + throws IOException, ParseException { + SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + DeleteResponse response = dao.delete(securityContext.getUserPrincipal().getName(), id); + return response.toResponse(); + } + + private GlossaryTerm getGlossaryTerm(SecurityContext securityContext, CreateGlossaryTerm create) throws IOException { + EntityReference glossary = Entity.getEntityReference(Entity.GLOSSARY, create.getGlossaryId()); + EntityReference parentTerm = + create.getParentId() != null ? Entity.getEntityReference(Entity.GLOSSARY_TERM, create.getParentId()) : null; + return new GlossaryTerm() + .withId(UUID.randomUUID()) + .withName(create.getName()) + .withDisplayName(create.getDisplayName()) + .withDescription(create.getDescription()) + .withSynonyms(create.getSynonyms()) + .withGlossary(glossary) + .withParent(parentTerm) + .withRelatedTerms(create.getRelatedTerms()) + .withReferences(create.getReferences()) + .withReviewers(create.getReviewers()) + .withTags(create.getTags()) + .withUpdatedBy(securityContext.getUserPrincipal().getName()) + .withUpdatedAt(System.currentTimeMillis()); + } +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java index c9565d72ce3..c353c1d8165 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java @@ -153,6 +153,9 @@ public class SearchResource { case "team_search_index": searchSourceBuilder = buildTeamSearchBuilder(query, from, size); break; + case "glossary_search_index": + searchSourceBuilder = buildGlossarySearchBuilder(query, from, size); + break; default: searchSourceBuilder = buildAggregateSearchBuilder(query, from, size); break; @@ -380,4 +383,27 @@ public class SearchResource { return searchSourceBuilder; } + + private SearchSourceBuilder buildGlossarySearchBuilder(String query, int from, int size) { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + HighlightBuilder.Field highlightGlossaryName = new HighlightBuilder.Field("glossary_name"); + highlightGlossaryName.highlighterType("unified"); + HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field("description"); + highlightDescription.highlighterType("unified"); + HighlightBuilder hb = new HighlightBuilder(); + hb.field(highlightDescription); + hb.field(highlightGlossaryName); + hb.preTags(""); + hb.postTags(""); + searchSourceBuilder + .query(QueryBuilders.queryStringQuery(query).field("glossary_name", 5.0f).field("description").lenient(true)) + .aggregation(AggregationBuilders.terms("EntityType").field("entity_type")) + .aggregation(AggregationBuilders.terms("Tier").field("tier")) + .aggregation(AggregationBuilders.terms("Tags").field("tags")) + .highlighter(hb) + .from(from) + .size(size); + + return searchSourceBuilder; + } } diff --git a/catalog-rest-service/src/main/resources/json/schema/api/data/createGlossary.json b/catalog-rest-service/src/main/resources/json/schema/api/data/createGlossary.json new file mode 100644 index 00000000000..e1feb966c72 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/api/data/createGlossary.json @@ -0,0 +1,41 @@ +{ + "$id": "https://open-metadata.org/schema/api/data/createGlossary.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Create Glossary entity request", + "description": "Create Glossary entity request", + "type": "object", + "properties" : { + "name": { + "description": "Name that identifies this glossary.", + "$ref": "../../entity/data/glossary.json#/definitions/name" + }, + "displayName": { + "description": "Display Name that identifies this glossary.", + "type": "string" + }, + "description": { + "description": "Description of the glossary instance.", + "type": "string" + }, + "reviewers": { + "description": "User names of the reviewers for this glossary.", + "type": "array", + "items" : { + "type": "string" + } + }, + "owner": { + "description": "Owner of this glossary", + "$ref": "../../type/entityReference.json" + }, + "tags": { + "description": "Tags for this glossary", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + } + }, + "required": ["name"] + } \ No newline at end of file diff --git a/catalog-rest-service/src/main/resources/json/schema/api/data/createGlossaryTerm.json b/catalog-rest-service/src/main/resources/json/schema/api/data/createGlossaryTerm.json new file mode 100644 index 00000000000..24ebee244d9 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/api/data/createGlossaryTerm.json @@ -0,0 +1,73 @@ +{ + "$id": "https://open-metadata.org/schema/api/data/createGlossary.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Create Glossary entity request", + "description": "Create Glossary entity request", + "type": "object", + "properties": { + "glossaryId" : { + "description": "UUID of the glossary.", + "$ref" : "../../type/basic.json#/definitions/uuid" + }, + "parentId" : { + "description": "UUID of the parent term. When null, the term is at the root of the glossary.", + "$ref" : "../../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Preferred name for the glossary term.", + "$ref": "../../entity/data/glossaryTerm.json#/definitions/name" + }, + "displayName": { + "description": "Display Name that identifies this glossary.", + "type": "string" + }, + "description": { + "description": "Description of the glossary term.", + "type": "string" + }, + "synonyms": { + "description": "Alternate names that are synonyms or near-synonyms for the glossary term.", + "type": "array", + "items": { + "$ref": "../../entity/data/glossaryTerm.json#/definitions/name" + } + }, + "children": { + "description": "Other glossary terms that are children of this glossary term.", + "type": "array", + "items": { + "$ref": "../../type/entityReference.json" + } + }, + "relatedTerms": { + "description": "Other glossary terms that are related to this glossary term.", + "type": "array", + "items": { + "$ref": "../../type/entityReference.json" + } + }, + "references": { + "description": "Link to a reference from an external glossary.", + "$ref": "../../entity/data/glossaryTerm.json#/definitions/termReference" + }, + "reviewers": { + "description": "User names of the reviewers for this glossary.", + "type": "array", + "items": { + "type": "string" + } + }, + "tags": { + "description": "Tags for this glossary term.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + } + }, + "required": [ + "glossaryId", + "name" + ] +} \ No newline at end of file diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/glossary.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/glossary.json new file mode 100644 index 00000000000..c4c5daa75f8 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/glossary.json @@ -0,0 +1,82 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/glossary.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Glossary", + "description": "This schema defines the Glossary entity based on SKOS.", + "type": "object", + "definitions": { + "name": { + "description": "Name that identifies a glossary term.", + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "properties": { + "id": { + "description": "Unique identifier of a glossary instance.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Preferred name for the glossary term.", + "type": "string", + "$ref": "#/definitions/name" + }, + "displayName": { + "description": "Display Name that identifies this glossary.", + "type": "string" + }, + "description": { + "description": "Description of the glossary.", + "type": "string" + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/entityVersion" + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "href": { + "description": "Link to the resource corresponding to this entity.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "reviewers": { + "description": "User names of the reviewers for this glossary.", + "type": "array", + "items": { + "type": "string" + } + }, + "owner": { + "description": "Owner of this glossary.", + "$ref": "../../type/entityReference.json" + }, + "tags": { + "description": "Tags for this glossary.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "changeDescription": { + "description": "Change that lead to this version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "deleted": { + "description": "When `true` indicates the entity has been soft deleted.", + "type": "boolean", + "default": false + } + }, + "required": [ + "id", + "name" + ] +} \ No newline at end of file diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/glossaryTerm.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/glossaryTerm.json new file mode 100644 index 00000000000..de5022a703f --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/glossaryTerm.json @@ -0,0 +1,129 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/glossaryTerm.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "GlossaryTerm", + "description": "This schema defines the Glossary term entities.", + "type": "object", + "definitions": { + "name": { + "description": "Name that identifies a glossary term.", + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "termReference": { + "type": "object", + "properties": { + "name": { + "description": "Name that identifies the source of an external glossary term. Example `HealthCare.gov`", + "type": "string" + }, + "endpoint": { + "description": "Name that identifies the source of an external glossary term. Example `HealthCare.gov`", + "type": "string", + "format": "uri" + } + } + } + }, + "properties": { + "id": { + "description": "Unique identifier of a glossary term instance.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Preferred name for the glossary term.", + "$ref": "#/definitions/name" + }, + "displayName": { + "description": "Display Name that identifies this glossary.", + "type": "string" + }, + "description": { + "description": "Description of the glossary term.", + "type": "string" + }, + "fullyQualifiedName": { + "description": "A unique name that identifies a glossary term. It captures name hierarchy of glossary of terms in the form of `glossaryName.parentTerm.childTerm`.", + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "synonyms": { + "description": "Alternate names that are synonyms or near-synonyms for the glossary term.", + "type": "array", + "items": { + "$ref": "#/definitions/name" + } + }, + "glossary": { + "description": "Glosary that this term belongs to.", + "$ref" : "../../type/entityReference.json" + }, + "parent": { + "description": "Parent glossary term that this term is child of. When `null` this term is the root term of the glossary.", + "$ref" : "../../type/entityReference.json" + }, + "children": { + "description": "Other glossary terms that are children of this glossary term.", + "$ref": "../../type/entityReference.json#/definitions/entityReferenceList" + }, + "relatedTerms": { + "description": "Other glossary terms that are related to this glossary term.", + "$ref": "../../type/entityReference.json#/definitions/entityReferenceList" + }, + "references": { + "description": "Link to a reference from an external glossary.", + "$ref": "#/definitions/termReference" + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/entityVersion" + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "skos": { + "description": "SKOS data in JSON-LD format", + "type": "string" + }, + "href": { + "description": "Link to the resource corresponding to this entity.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "reviewers": { + "description": "User names of the reviewers for this glossary.", + "type": "array", + "items": { + "type": "string" + } + }, + "tags": { + "description": "Tags for this glossary term.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "changeDescription": { + "description": "Change that lead to this version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "deleted": { + "description": "When `true` indicates the entity has been soft deleted.", + "type": "boolean", + "default": false + } + }, + "required": [ + "id", + "name", + "glossary" + ] +} \ No newline at end of file diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java index e19ebd202ea..21faf77034a 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java @@ -829,15 +829,13 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { // Create an entity with owner K request = createRequest(getEntityName(test), "description", "displayName", USER_OWNER1); - createAndCheckEntity(request, ADMIN_AUTH_HEADERS); + createEntity(request, ADMIN_AUTH_HEADERS); // Update description and remove owner as non-owner // Expect to throw an exception since only owner or admin can update resource - K updateRequest = createRequest(getEntityName(test), "newdescription", "displayName", null); + K updateRequest = createRequest(getEntityName(test), "newDescription", "displayName", null); HttpResponseException exception = - assertThrows( - HttpResponseException.class, - () -> updateAndCheckEntity(updateRequest, OK, TEST_AUTH_HEADERS, UpdateType.NO_CHANGE, null)); + assertThrows(HttpResponseException.class, () -> updateEntity(updateRequest, OK, TEST_AUTH_HEADERS)); TestUtils.assertResponse( exception, FORBIDDEN, "Principal: CatalogPrincipal{name='test'} " + "does not have permissions"); } @@ -846,7 +844,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { void put_entityNullDescriptionUpdate_200(TestInfo test) throws IOException { // Create entity with null description K request = createRequest(getEntityName(test), null, "displayName", null); - T entity = createAndCheckEntity(request, ADMIN_AUTH_HEADERS); + T entity = createEntity(request, ADMIN_AUTH_HEADERS); EntityInterface entityInterface = getEntityInterface(entity); // Update null description with a new description @@ -861,7 +859,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { void put_entityEmptyDescriptionUpdate_200(TestInfo test) throws IOException { // Create entity with empty description K request = createRequest(getEntityName(test), "", "displayName", null); - T entity = createAndCheckEntity(request, ADMIN_AUTH_HEADERS); + T entity = createEntity(request, ADMIN_AUTH_HEADERS); EntityInterface entityInterface = getEntityInterface(entity); // Update empty description with a new description diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryResourceTest.java new file mode 100644 index 00000000000..946844e61d1 --- /dev/null +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryResourceTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.glossary; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Map; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestMethodOrder; +import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.api.data.CreateGlossary; +import org.openmetadata.catalog.entity.data.Glossary; +import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface; +import org.openmetadata.catalog.resources.EntityResourceTest; +import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.util.TestUtils; + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class GlossaryResourceTest extends EntityResourceTest { + public GlossaryResourceTest() { + super( + Entity.GLOSSARY, + Glossary.class, + GlossaryResource.GlossaryList.class, + "glossaries", + GlossaryResource.FIELDS, + false, + true, + true, + true); + } + + @BeforeAll + public void setup(TestInfo test) throws IOException, URISyntaxException { + super.setup(test); + } + + @Override + public CreateGlossary createRequest(String name, String description, String displayName, EntityReference owner) { + return new CreateGlossary() + .withName(name) + .withDescription(description) + .withDisplayName(displayName) + .withOwner(owner); + } + + @Override + public EntityReference getContainer(CreateGlossary createRequest) { + return null; + } + + @Override + public void validateCreatedEntity( + Glossary createdEntity, CreateGlossary createRequest, Map authHeaders) + throws HttpResponseException { + validateCommonEntityFields( + getEntityInterface(createdEntity), + createRequest.getDescription(), + TestUtils.getPrincipal(authHeaders), + createRequest.getOwner()); + + // Entity specific validation + TestUtils.validateTags(createRequest.getTags(), createdEntity.getTags()); + } + + @Override + public void validateUpdatedEntity(Glossary updated, CreateGlossary request, Map authHeaders) + throws HttpResponseException { + validateCreatedEntity(updated, request, authHeaders); + } + + @Override + public void compareEntities(Glossary expected, Glossary patched, Map authHeaders) + throws HttpResponseException { + validateCommonEntityFields( + getEntityInterface(patched), + expected.getDescription(), + TestUtils.getPrincipal(authHeaders), + expected.getOwner()); + + // Entity specific validation + TestUtils.validateTags(expected.getTags(), patched.getTags()); + } + + @Override + public GlossaryEntityInterface getEntityInterface(Glossary entity) { + return new GlossaryEntityInterface(entity); + } + + @Override + public void validateGetWithDifferentFields(Glossary entity, boolean byName) throws HttpResponseException {} + + @Override + public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException { + if (expected == actual) { + return; + } + assertCommonFieldChange(fieldName, expected, actual); + } +} diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResourceTest.java new file mode 100644 index 00000000000..07b0b5a0f6c --- /dev/null +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResourceTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.glossary; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS; +import static org.openmetadata.catalog.util.TestUtils.validateEntityReference; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Map; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestMethodOrder; +import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.api.data.CreateGlossary; +import org.openmetadata.catalog.api.data.CreateGlossaryTerm; +import org.openmetadata.catalog.entity.data.Glossary; +import org.openmetadata.catalog.entity.data.GlossaryTerm; +import org.openmetadata.catalog.jdbi3.GlossaryTermRepository.GlossaryTermEntityInterface; +import org.openmetadata.catalog.resources.EntityResourceTest; +import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.util.TestUtils; + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class GlossaryTermResourceTest extends EntityResourceTest { + public static Glossary GLOSSARY; + + public GlossaryTermResourceTest() { + super( + Entity.GLOSSARY_TERM, + GlossaryTerm.class, + GlossaryTermResource.GlossaryTermList.class, + "glossaryTerms", + GlossaryTermResource.FIELDS, + false, + false, + true, + false); + } + + @BeforeAll + public void setup(TestInfo test) throws IOException, URISyntaxException { + super.setup(test); + GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest(); + CreateGlossary createGlossary = glossaryResourceTest.createRequest(test); + GLOSSARY = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS); + } + + @Override + public CreateGlossaryTerm createRequest(String name, String description, String displayName, EntityReference owner) { + return new CreateGlossaryTerm() + .withName(name) + .withDescription(description) + .withDisplayName(displayName) + .withGlossaryId(GLOSSARY.getId()); + } + + @Override + public EntityReference getContainer(CreateGlossaryTerm createRequest) { + return null; + } + + /** + * A method variant to be called form other tests to create a glossary without depending on Database, DatabaseService + * set up in the {@code setup()} method + */ + public GlossaryTerm createEntity(TestInfo test, int index) throws IOException { + CreateGlossaryTerm create = new CreateGlossaryTerm().withName(getEntityName(test, index)); + return createEntity(create, ADMIN_AUTH_HEADERS); + } + + @Override + public void validateCreatedEntity( + GlossaryTerm createdEntity, CreateGlossaryTerm createRequest, Map authHeaders) + throws HttpResponseException { + validateCommonEntityFields( + getEntityInterface(createdEntity), createRequest.getDescription(), TestUtils.getPrincipal(authHeaders), null); + + validateEntityReference(createdEntity.getGlossary()); + assertEquals(createRequest.getGlossaryId(), createdEntity.getGlossary().getId()); + + // Entity specific validation + TestUtils.validateTags(createRequest.getTags(), createdEntity.getTags()); + } + + @Override + public void validateUpdatedEntity(GlossaryTerm updated, CreateGlossaryTerm request, Map authHeaders) + throws HttpResponseException { + validateCreatedEntity(updated, request, authHeaders); + } + + @Override + public void compareEntities(GlossaryTerm expected, GlossaryTerm patched, Map authHeaders) + throws HttpResponseException { + validateCommonEntityFields( + getEntityInterface(patched), expected.getDescription(), TestUtils.getPrincipal(authHeaders), null); + + validateEntityReference(patched.getGlossary()); + assertEquals(expected.getGlossary().getId(), patched.getGlossary().getId()); + + // Entity specific validation + TestUtils.validateTags(expected.getTags(), patched.getTags()); + } + + @Override + public GlossaryTermEntityInterface getEntityInterface(GlossaryTerm entity) { + return new GlossaryTermEntityInterface(entity); + } + + @Override + public void validateGetWithDifferentFields(GlossaryTerm entity, boolean byName) throws HttpResponseException {} + + @Override + public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException { + if (expected == actual) { + return; + } + assertCommonFieldChange(fieldName, expected, actual); + } +} diff --git a/ingestion/examples/sample_data/thesauruses/hobbies-SKOS.json b/ingestion/examples/sample_data/thesauruses/hobbies-SKOS.json new file mode 100644 index 00000000000..8930ecbdf52 --- /dev/null +++ b/ingestion/examples/sample_data/thesauruses/hobbies-SKOS.json @@ -0,0 +1,1332 @@ +[ + { + "@id": "http://hobbies.com/achievement_hobbies", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Achievement hobbies", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#topConceptOf": [ + { + "@id": "http://hobbies.com/scheme" + } + ] + }, + { + "@id": "http://hobbies.com/air_sports", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Air sports", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/airsoft", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Airsoft", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/amateur_radio", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Amateur radio", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/antiquities", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Antiquities", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/archery", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Archery", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/audiophilia", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Audiophilia", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/auto_audiophilia", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Auto audiophilia", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/auto_racing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Auto racing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/badminton", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Badminton", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/baseball", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Baseball", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/basketball", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Basketball", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/baton_twirling", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Baton twirling", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/billiards", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Billiards", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/board_sports", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Board sports", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/bonsai", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Bonsai", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/bowling", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Bowling", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/boxing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Boxing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/chess", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Chess", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/climbing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Climbing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/coin_collecting", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Coin collecting", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/collection_hobbies", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Collection hobbies", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#topConceptOf": [ + { + "@id": "http://hobbies.com/scheme" + } + ] + }, + { + "@id": "http://hobbies.com/competition_hobbies", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Competition hobbies", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#topConceptOf": [ + { + "@id": "http://hobbies.com/scheme" + } + ] + }, + { + "@id": "http://hobbies.com/computer_programming", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Computer programming", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/cricket", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Cricket", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/cycling", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Cycling", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/dance", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Dance", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/darts", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Darts", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/disc_golf", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Disc golf", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/equestrianism", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Equestrianism", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/fencing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Fencing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/figure_skating", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Figure skating", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/fishing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Fishing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/fossil_collecting", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Fossil collecting", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/gardening", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Gardening", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/golfing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Golfing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/gymnastics", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Gymnastics", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/handball", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Handball", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/home_automation", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Home automation", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/ice_hockey", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Ice hockey", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/indoors", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Collection" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "indoors", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#member": [ + { + "@id": "http://hobbies.com/audiophilia" + }, + { + "@id": "http://hobbies.com/baton_twirling" + }, + { + "@id": "http://hobbies.com/bonsai" + }, + { + "@id": "http://hobbies.com/computer_programming" + }, + { + "@id": "http://hobbies.com/dance" + }, + { + "@id": "http://hobbies.com/amateur_radio" + }, + { + "@id": "http://hobbies.com/home_automation" + }, + { + "@id": "http://hobbies.com/knapping" + }, + { + "@id": "http://hobbies.com/lapidary" + }, + { + "@id": "http://hobbies.com/locksport" + }, + { + "@id": "http://hobbies.com/scrapbooking" + }, + { + "@id": "http://hobbies.com/coin_collecting" + }, + { + "@id": "http://hobbies.com/stamp_collecting" + }, + { + "@id": "http://hobbies.com/billiards" + }, + { + "@id": "http://hobbies.com/bowling" + }, + { + "@id": "http://hobbies.com/boxing" + }, + { + "@id": "http://hobbies.com/chess" + }, + { + "@id": "http://hobbies.com/darts" + }, + { + "@id": "http://hobbies.com/fencing" + }, + { + "@id": "http://hobbies.com/role-playing_games" + }, + { + "@id": "http://hobbies.com/table_football" + }, + { + "@id": "http://hobbies.com/handball" + }, + { + "@id": "http://hobbies.com/martial_arts" + }, + { + "@id": "http://hobbies.com/video_gaming" + } + ] + }, + { + "@id": "http://hobbies.com/kart_racing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Kart racing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/kayaking", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Kayaking", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/kitesurfing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Kitesurfing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/knapping", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Knapping", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/lapidary", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Lapidary", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/locksport", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Locksport", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/martial_arts", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Martial arts", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/metal_detecting", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Metal detecting", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/mineral_collecting", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Mineral collecting", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/model_building", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Model building", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/model_rocketry", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Model rocketry", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/model_building" + } + ] + }, + { + "@id": "http://hobbies.com/mountainbiking", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Mountainbiking", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/outdoors", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Collection" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "outdoors", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#member": [ + { + "@id": "http://hobbies.com/air_sports" + }, + { + "@id": "http://hobbies.com/board_sports" + }, + { + "@id": "http://hobbies.com/kayaking" + }, + { + "@id": "http://hobbies.com/kitesurfing" + }, + { + "@id": "http://hobbies.com/mountainbiking" + }, + { + "@id": "http://hobbies.com/scuba_diving" + }, + { + "@id": "http://hobbies.com/snowmobiling" + }, + { + "@id": "http://hobbies.com/antiquities" + }, + { + "@id": "http://hobbies.com/auto_audiophilia" + }, + { + "@id": "http://hobbies.com/fossil_collecting" + }, + { + "@id": "http://hobbies.com/metal_detecting" + }, + { + "@id": "http://hobbies.com/mineral_collecting" + }, + { + "@id": "http://hobbies.com/airsoft" + }, + { + "@id": "http://hobbies.com/archery" + }, + { + "@id": "http://hobbies.com/auto_racing" + }, + { + "@id": "http://hobbies.com/badminton" + }, + { + "@id": "http://hobbies.com/baseball" + }, + { + "@id": "http://hobbies.com/basketball" + }, + { + "@id": "http://hobbies.com/climbing" + }, + { + "@id": "http://hobbies.com/cricket" + }, + { + "@id": "http://hobbies.com/cycling" + }, + { + "@id": "http://hobbies.com/disc_golf" + }, + { + "@id": "http://hobbies.com/equestrianism" + }, + { + "@id": "http://hobbies.com/figure_skating" + }, + { + "@id": "http://hobbies.com/fishing" + }, + { + "@id": "http://hobbies.com/golfing" + }, + { + "@id": "http://hobbies.com/gymnastics" + }, + { + "@id": "http://hobbies.com/ice_hockey" + }, + { + "@id": "http://hobbies.com/kart_racing" + }, + { + "@id": "http://hobbies.com/paintball" + }, + { + "@id": "http://hobbies.com/rugby_league" + }, + { + "@id": "http://hobbies.com/swimming" + }, + { + "@id": "http://hobbies.com/tennis" + } + ] + }, + { + "@id": "http://hobbies.com/paintball", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Paintball", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/photography", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Photography", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/radio-controlled_modeling", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Radio-controlled modeling", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/model_building" + } + ] + }, + { + "@id": "http://hobbies.com/role-playing_games", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Role-playing games", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/rugby_league", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Rugby league", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/scale_modeling", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Scale modeling", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/model_building" + } + ] + }, + { + "@id": "http://hobbies.com/scheme" + }, + { + "@id": "http://hobbies.com/scrapbooking", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Scrapbooking", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/scuba_diving", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Scuba diving", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/snowmobiling", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Snowmobiling", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/stamp_collecting", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Stamp collecting", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/swimming", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Swimming", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/table_football", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Table football", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/tennis", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Tennis", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/video_gaming", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Video gaming", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://www.w3.org/2004/02/skos/core#Collection" + }, + { + "@id": "http://www.w3.org/2004/02/skos/core#Concept" + } + ] \ No newline at end of file diff --git a/ingestion/pipelines/metadata_to_es.json b/ingestion/pipelines/metadata_to_es.json index a95361a838f..c459523c1a5 100644 --- a/ingestion/pipelines/metadata_to_es.json +++ b/ingestion/pipelines/metadata_to_es.json @@ -5,6 +5,7 @@ "include_tables": "true", "include_topics": "true", "include_dashboards": "true", + "include_glossary": "true", "limit_records": 10 } }, @@ -14,6 +15,7 @@ "index_tables": "true", "index_topics": "true", "index_dashboards": "true", + "index_glossary": "true", "es_host": "localhost", "es_port": 9200 } diff --git a/ingestion/src/metadata/ingestion/models/table_metadata.py b/ingestion/src/metadata/ingestion/models/table_metadata.py index df9ecf36e8b..adb3041c963 100644 --- a/ingestion/src/metadata/ingestion/models/table_metadata.py +++ b/ingestion/src/metadata/ingestion/models/table_metadata.py @@ -104,6 +104,23 @@ class TopicESDocument(BaseModel): doc_as_upsert: bool = True +class GlossaryESDocument(BaseModel): + """Glossary Elastic Search Mapping doc""" + + glossary_id: str + glossary_name: str + entity_type: str = "glossary" + suggest: List[dict] + description: Optional[str] = None + last_updated_timestamp: Optional[int] + tags: List[str] + fqdn: str + tier: Optional[str] = None + schema_description: Optional[str] = None + owner: str + followers: List[str] + + class DashboardESDocument(BaseModel): """Elastic Search Mapping doc for Dashboards""" diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 616c0284672..c889c9eb77c 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -25,6 +25,7 @@ from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.entity.data.chart import Chart from metadata.generated.schema.entity.data.dashboard import Dashboard from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.glossary import Glossary from metadata.generated.schema.entity.data.location import Location from metadata.generated.schema.entity.data.metrics import Metrics from metadata.generated.schema.entity.data.mlmodel import MlModel @@ -169,6 +170,11 @@ class OpenMetadata( ): return "/mlmodels" + if issubclass( + entity, get_args(Union[Glossary, self.get_create_entity_type(Glossary)]) + ): + return "/glossary" + if issubclass( entity, get_args(Union[Chart, self.get_create_entity_type(Chart)]) ): diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch.py b/ingestion/src/metadata/ingestion/sink/elasticsearch.py index 5009303c141..6b229b7e32f 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch.py @@ -24,6 +24,7 @@ from metadata.config.common import ConfigModel from metadata.generated.schema.entity.data.chart import Chart from metadata.generated.schema.entity.data.dashboard import Dashboard from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.glossary import Glossary from metadata.generated.schema.entity.data.pipeline import Pipeline, Task from metadata.generated.schema.entity.data.table import Column, Table from metadata.generated.schema.entity.data.topic import Topic @@ -39,6 +40,7 @@ from metadata.ingestion.api.sink import Sink, SinkStatus from metadata.ingestion.models.table_metadata import ( ChangeDescription, DashboardESDocument, + GlossaryESDocument, PipelineESDocument, TableESDocument, TeamESDocument, @@ -49,6 +51,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig from metadata.ingestion.sink.elasticsearch_constants import ( DASHBOARD_ELASTICSEARCH_INDEX_MAPPING, + GLOSSARY_ELASTICSEARCH_INDEX_MAPPING, PIPELINE_ELASTICSEARCH_INDEX_MAPPING, TABLE_ELASTICSEARCH_INDEX_MAPPING, TEAM_ELASTICSEARCH_INDEX_MAPPING, @@ -70,11 +73,13 @@ class ElasticSearchConfig(ConfigModel): es_password: Optional[str] = None index_tables: Optional[bool] = True index_topics: Optional[bool] = True + index_glossary: Optional[bool] = True index_dashboards: Optional[bool] = True index_pipelines: Optional[bool] = True index_users: Optional[bool] = True index_teams: Optional[bool] = True table_index_name: str = "table_search_index" + glossary_index_name: str = "glossary_search_index" topic_index_name: str = "topic_search_index" dashboard_index_name: str = "dashboard_search_index" pipeline_index_name: str = "pipeline_search_index" @@ -139,6 +144,10 @@ class ElasticsearchSink(Sink[Entity]): self._check_or_create_index( self.config.table_index_name, TABLE_ELASTICSEARCH_INDEX_MAPPING ) + if self.config.index_glossary: + self._check_or_create_index( + self.config.glossary_index_name, GLOSSARY_ELASTICSEARCH_INDEX_MAPPING + ) if self.config.index_topics: self._check_or_create_index( self.config.topic_index_name, TOPIC_ELASTICSEARCH_INDEX_MAPPING @@ -201,6 +210,13 @@ class ElasticsearchSink(Sink[Entity]): body=table_doc.json(), request_timeout=self.config.timeout, ) + if isinstance(record, Glossary): + glossary_doc = self._create_glossary_es_doc(record) + self.elasticsearch_client.index( + index=self.config.glossary_index_name, + id=str(glossary_doc.glossary_id), + body=glossary_doc.json(), + ) if isinstance(record, Topic): topic_doc = self._create_topic_es_doc(record) self.elasticsearch_client.index( @@ -538,6 +554,61 @@ class ElasticsearchSink(Sink[Entity]): return team_doc + def _create_glossary_es_doc(self, glossary: Glossary): + fqdn = glossary.fullyQualifiedName + suggest = [ + { + "input": [ + glossary.displayName if glossary.displayName else glossary.name + ], + "weight": 10, + } + ] + tags = set() + timestamp = time.time() + glossary_owner = ( + str(glossary.owner.id.__root__) if glossary.owner is not None else "" + ) + glossary_followers = [] + if glossary.followers: + for follower in glossary.followers.__root__: + glossary_followers.append(str(follower.id.__root__)) + tier = None + for glossary_tag in glossary.tags: + if "Tier" in glossary_tag.tagFQN: + tier = glossary_tag.tagFQN + else: + tags.add(glossary_tag.tagFQN) + # tasks: List[Task] = glossary.tasks # TODO Handle Glossary words + # task_names = [] + # task_descriptions = [] + # for task in tasks: + # task_names.append(task.displayName) + # if task.description is not None: + # task_descriptions.append(task.description) + # if tags in task and len(task.tags) > 0: + # for col_tag in task.tags: + # tags.add(col_tag.tagFQN) + + glossary_doc = GlossaryESDocument( + glossary_id=str(glossary.id.__root__), + glossary_name=glossary.displayName + if glossary.displayName + else glossary.name, + # task_names=task_names, # TODO Handle Glossary words + # task_descriptions=task_descriptions, + suggest=suggest, + description=glossary.description, + last_updated_timestamp=timestamp, + tier=tier, + tags=list(tags), + fqdn=fqdn, + owner=glossary_owner, + followers=glossary_followers, + ) + + return glossary_doc + def _get_charts(self, chart_refs: Optional[List[entityReference.EntityReference]]): charts = [] if chart_refs: diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py index 611d158ac95..7b87d6a8cb5 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py @@ -200,6 +200,48 @@ TOPIC_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( """ ) +GLOSSARY_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( + """ + { + "mappings":{ + "properties": { + "glossary_name": { + "type":"text" + }, + "display_name": { + "type": "text" + }, + "owner": { + "type": "text" + }, + "followers": { + "type": "keyword" + }, + "last_updated_timestamp": { + "type": "date", + "format": "epoch_second" + }, + "description": { + "type": "text" + }, + "tier": { + "type": "keyword" + }, + "tags": { + "type": "keyword" + }, + "entity_type": { + "type": "keyword" + }, + "suggest": { + "type": "completion" + } + } + } + } + """ +) + DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( """ { diff --git a/ingestion/src/metadata/ingestion/source/metadata.py b/ingestion/src/metadata/ingestion/source/metadata.py index aa58b86e321..8a0b44b7343 100644 --- a/ingestion/src/metadata/ingestion/source/metadata.py +++ b/ingestion/src/metadata/ingestion/source/metadata.py @@ -24,7 +24,7 @@ from metadata.generated.schema.entity.teams.user import User from metadata.ingestion.api.common import Entity, WorkflowContext from metadata.ingestion.api.source import Source, SourceStatus from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig +from metadata.ingestion.ometa.openmetadata_rest import Glossary, MetadataServerConfig logger = logging.getLogger(__name__) @@ -35,6 +35,7 @@ class MetadataTablesRestSourceConfig(ConfigModel): include_tables: Optional[bool] = True include_topics: Optional[bool] = True include_dashboards: Optional[bool] = True + include_glossary: Optional[bool] = True include_pipelines: Optional[bool] = True include_users: Optional[bool] = True include_teams: Optional[bool] = True @@ -64,6 +65,10 @@ class MetadataSourceStatus(SourceStatus): self.success.append(table_name) logger.info("Table Scanned: %s", table_name) + def scanned_glossary(self, glossary_name: str) -> None: + self.success.append(glossary_name) + logger.info("Glossary Scanned: {}".format(glossary_name)) + def scanned_topic(self, topic_name: str) -> None: """scanned topic method @@ -165,6 +170,7 @@ class MetadataSource(Source[Entity]): def next_record(self) -> Iterable[Entity]: yield from self.fetch_table() yield from self.fetch_topic() + yield from self.fetch_glossary() yield from self.fetch_dashboard() yield from self.fetch_pipeline() yield from self.fetch_users() @@ -315,6 +321,23 @@ class MetadataSource(Source[Entity]): break after = team_entities.after + def fetch_glossary(self) -> Glossary: + if self.config.include_glossary: + after = None + while True: + glossary_entities = self.metadata.list_entities( + entity=Glossary, + fields=["owner", "tags", "followers"], + after=after, + limit=self.config.limit_records, + ) + for glossary in glossary_entities.entities: + self.status.scanned_glossary(glossary.name) + yield glossary + if glossary_entities.after is None: + break + after = glossary_entities.after + def get_status(self) -> SourceStatus: return self.status diff --git a/ingestion/tests/unit/test_ometa_endpoints.py b/ingestion/tests/unit/test_ometa_endpoints.py index a93994cbbde..dcf9e1a6401 100644 --- a/ingestion/tests/unit/test_ometa_endpoints.py +++ b/ingestion/tests/unit/test_ometa_endpoints.py @@ -27,6 +27,7 @@ from metadata.generated.schema.entity.data.mlmodel import MlModel from metadata.generated.schema.entity.data.pipeline import Pipeline from metadata.generated.schema.entity.data.report import Report from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.data.glossary import Glossary from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.services.dashboardService import DashboardService from metadata.generated.schema.entity.services.databaseService import DatabaseService @@ -51,6 +52,10 @@ class OMetaEndpointTest(TestCase): Pass Entities and test their suffix generation """ + + # Glossary + self.assertEqual(self.metadata.get_suffix(Glossary), "/glossary") + # ML self.assertEqual(self.metadata.get_suffix(MlModel), "/mlmodels")