yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-07 21:16:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #2760 - Add entities for Glossary and initial API (#2761)

Browse Source 
* Fixes #2760 - Add entities for Glossary and initial API

* Fixing merge issues
This commit is contained in:
Suresh Srinivas 2022-02-15 20:54:46 -08:00 committed by GitHub
parent 56a45faf1c
commit 28ba1a3c04
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 3638 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
bootstrap/sql/mysql/v004__create_db_connection_info.sql
View File

@ -15,3 +15,27 @@ CREATE TABLE IF NOT EXISTS thread_entity (
INDEX (updatedAt)
);
CREATE TABLE IF NOT EXISTS glossary_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(name),
INDEX (updatedBy),
INDEX (updatedAt)
);
CREATE TABLE IF NOT EXISTS glossary_term_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
);

22
catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java
View File

@ -40,9 +40,16 @@ import org.openmetadata.catalog.util.EntityUtil.Fields;
@Slf4j
public final class Entity {
private static final Map<String, EntityDAO<?>> DAO_MAP = new HashMap<>();
private static final Map<String, EntityRepository<?>> ENTITY_REPOSITORY_MAP = new HashMap<>();
// Lower case entity name to canonical entity name map
private static final Map<String, String> CANONICAL_ENTITY_NAME_MAP = new HashMap<>();
// Canonical entity name to corresponding EntityDAO map
private static final Map<String, EntityDAO<?>> DAO_MAP = new HashMap<>();
// Canonical entity name to corresponding EntityRepository map
private static final Map<String, EntityRepository<?>> ENTITY_REPOSITORY_MAP = new HashMap<>();
// Entity class to entity repository map
private static final Map<Class<?>, EntityRepository<?>> CLASS_ENTITY_REPOSITORY_MAP = new HashMap<>();
//
@ -70,6 +77,8 @@ public final class Entity {
public static final String UNUSED = "unused";
public static final String BOTS = "bots";
public static final String LOCATION = "location";
public static final String GLOSSARY = "glossary";
public static final String GLOSSARY_TERM = "glossaryTerm";
//
// Policies
@ -92,12 +101,17 @@ public final class Entity {
private Entity() {}
public static <T> void registerEntity(
Class clazz, String entity, EntityDAO<T> dao, EntityRepository<T> entityRepository) {
Class<T> clazz, String entity, EntityDAO<T> dao, EntityRepository<T> entityRepository) {
DAO_MAP.put(entity, dao);
ENTITY_REPOSITORY_MAP.put(entity, entityRepository);
CANONICAL_ENTITY_NAME_MAP.put(entity.toLowerCase(Locale.ROOT), entity);
CLASS_ENTITY_REPOSITORY_MAP.put(clazz, entityRepository);
LOG.info("Registering entity {}", entity);
LOG.info(
"Registering entity {} {} {} {}",
clazz,
entity,
dao.getEntityClass().getSimpleName(),
entityRepository.getClass().getSimpleName());
}
public static EntityReference getEntityReference(String entity, UUID id) throws IOException {

65
catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java
View File

@ -12,6 +12,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import javax.ws.rs.core.Response;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
@ -31,6 +32,7 @@ import org.elasticsearch.client.indices.PutMappingRequest;
import org.elasticsearch.common.xcontent.XContentType;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.entity.data.Pipeline;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.entity.data.Topic;
@ -67,7 +69,8 @@ public class ElasticSearchIndexDefinition {
DASHBOARD_SEARCH_INDEX("dashboard_search_index", "/elasticsearch/dashboard_index_mapping.json"),
PIPELINE_SEARCH_INDEX("pipeline_search_index", "/elasticsearch/pipeline_index_mapping.json"),
USER_SEARCH_INDEX("user_search_index", "/elasticsearch/user_index_mapping.json"),
TEAM_SEARCH_INDEX("team_search_index", "/elasticsearch/team_index_mapping.json");
TEAM_SEARCH_INDEX("team_search_index", "/elasticsearch/team_index_mapping.json"),
GLOSSARY_SEARCH_INDEX("glossary_search_index", "/elasticsearch/glossary_index_mapping.json");
public final String indexName;
public final String indexMappingFile;
@ -197,6 +200,8 @@ public class ElasticSearchIndexDefinition {
return ElasticSearchIndexType.USER_SEARCH_INDEX;
} else if (type.equalsIgnoreCase(Entity.TEAM)) {
return ElasticSearchIndexType.TEAM_SEARCH_INDEX;
} else if (type.equalsIgnoreCase(Entity.GLOSSARY)) {
return ElasticSearchIndexType.GLOSSARY_SEARCH_INDEX;
}
throw new RuntimeException("Failed to find index doc for type " + type);
}
@ -852,3 +857,61 @@ class TeamESIndex {
return teamESIndexBuilder;
}
}
@EqualsAndHashCode(callSuper = true)
@Getter
@SuperBuilder(builderMethodName = "internalBuilder")
@Value
@JsonInclude(JsonInclude.Include.NON_NULL)
class GlossaryESIndex extends ElasticSearchIndex {
@JsonProperty("glossary_id")
String glossaryId;
public static GlossaryESIndexBuilder builder(Glossary glossary, int responseCode) {
List<String> tags = new ArrayList<>();
List<String> taskNames = new ArrayList<>();
List<String> taskDescriptions = new ArrayList<>();
List<ElasticSearchSuggest> suggest = new ArrayList<>();
suggest.add(ElasticSearchSuggest.builder().input(glossary.getName()).weight(5).build());
suggest.add(ElasticSearchSuggest.builder().input(glossary.getDisplayName()).weight(10).build());
if (glossary.getTags() != null) {
glossary.getTags().forEach(tag -> tags.add(tag.getTagFQN()));
}
Long updatedTimestamp = glossary.getUpdatedAt();
ParseTags parseTags = new ParseTags(tags);
String description = glossary.getDescription() != null ? glossary.getDescription() : "";
String displayName = glossary.getDisplayName() != null ? glossary.getDisplayName() : "";
GlossaryESIndexBuilder builder =
internalBuilder()
.glossaryId(glossary.getId().toString())
.name(glossary.getDisplayName())
.displayName(description)
.description(displayName)
.fqdn(glossary.getName())
.lastUpdatedTimestamp(updatedTimestamp)
.entityType("glossary")
.suggest(suggest)
.tags(parseTags.tags)
.tier(parseTags.tierTag);
if (glossary.getOwner() != null) {
builder.owner(glossary.getOwner().getId().toString());
}
ESChangeDescription esChangeDescription = null;
if (glossary.getChangeDescription() != null) {
esChangeDescription =
ESChangeDescription.builder().updatedAt(updatedTimestamp).updatedBy(glossary.getUpdatedBy()).build();
esChangeDescription.setFieldsAdded(glossary.getChangeDescription().getFieldsAdded());
esChangeDescription.setFieldsDeleted(glossary.getChangeDescription().getFieldsDeleted());
esChangeDescription.setFieldsUpdated(glossary.getChangeDescription().getFieldsUpdated());
} else if (responseCode == Response.Status.CREATED.getStatusCode()) {
esChangeDescription =
ESChangeDescription.builder().updatedAt(updatedTimestamp).updatedBy(glossary.getUpdatedBy()).build();
}
builder.changeDescriptions(esChangeDescription != null ? List.of(esChangeDescription) : null);
return builder;
}
}

59
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java
View File

@ -20,6 +20,7 @@ import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import org.jdbi.v3.core.mapper.RowMapper;
import org.jdbi.v3.core.statement.StatementContext;
import org.jdbi.v3.sqlobject.CreateSqlObject;
@ -33,6 +34,8 @@ import org.openmetadata.catalog.entity.Bots;
import org.openmetadata.catalog.entity.data.Chart;
import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.entity.data.Database;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.entity.data.GlossaryTerm;
import org.openmetadata.catalog.entity.data.Location;
import org.openmetadata.catalog.entity.data.Metrics;
import org.openmetadata.catalog.entity.data.MlModel;
@ -58,6 +61,8 @@ import org.openmetadata.catalog.jdbi3.DashboardRepository.DashboardEntityInterfa
import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface;
import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface;
import org.openmetadata.catalog.jdbi3.GlossaryTermRepository.GlossaryTermEntityInterface;
import org.openmetadata.catalog.jdbi3.LocationRepository.LocationEntityInterface;
import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.MetricsRepository.MetricsEntityInterface;
@ -135,6 +140,12 @@ public interface CollectionDAO {
@CreateSqlObject
MlModelDAO mlModelDAO();
@CreateSqlObject
GlossaryDAO glossaryDAO();
@CreateSqlObject
GlossaryTermDAO glossaryTermDAO();
@CreateSqlObject
BotsDAO botsDAO();
@ -331,6 +342,10 @@ public interface CollectionDAO {
}
interface EntityRelationshipDAO {
default int insert(UUID fromId, UUID toId, String fromEntity, String toEntity, int relation) {
return insert(fromId.toString(), toId.toString(), fromEntity, toEntity, relation);
}
@SqlUpdate(
"INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) "
+ "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation)")
@ -690,6 +705,50 @@ public interface CollectionDAO {
}
}
interface GlossaryDAO extends EntityDAO<Glossary> {
@Override
default String getTableName() {
return "glossary_entity";
}
@Override
default Class<Glossary> getEntityClass() {
return Glossary.class;
}
@Override
default String getNameColumn() {
return "name";
}
@Override
default EntityReference getEntityReference(Glossary entity) {
return new GlossaryEntityInterface(entity).getEntityReference();
}
}
interface GlossaryTermDAO extends EntityDAO<GlossaryTerm> {
@Override
default String getTableName() {
return "glossary_term_entity";
}
@Override
default Class<GlossaryTerm> getEntityClass() {
return GlossaryTerm.class;
}
@Override
default String getNameColumn() {
return "fullyQualifiedName";
}
@Override
default EntityReference getEntityReference(GlossaryTerm entity) {
return new GlossaryTermEntityInterface(entity).getEntityReference();
}
}
interface AirflowPipelineDAO extends EntityDAO<AirflowPipeline> {
@Override
default String getTableName() {

20
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java
View File

@ -30,7 +30,6 @@ import java.net.URI;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
@ -76,7 +75,6 @@ import org.openmetadata.catalog.util.ResultList;
import org.openmetadata.common.utils.CipherText;
import org.openmetadata.common.utils.CommonUtil;
@Slf4j
/**
* This is the base class used by Entity Resources to perform READ and WRITE operations to the backend database to
* Create, Retrieve, Update, and Delete entities.
@ -108,6 +106,7 @@ import org.openmetadata.common.utils.CommonUtil;
* relationship table when required to ensure, the data stored is efficiently and consistently, and relationship
* information does not become stale.
*/
@Slf4j
public abstract class EntityRepository<T> {
private final String collectionPath;
private final Class<T> entityClass;
@ -561,21 +560,18 @@ public abstract class EntityRepository<T> {
}
}
public static final Fields FIELDS_OWNER = new Fields(List.of("owner"), "owner");
public final EntityReference getOriginalOwner(T entity) throws IOException, ParseException {
final String FIELDS = "owner";
final List<String> FIELD_LIST = Arrays.asList(FIELDS.replace(" ", "").split(","));
EntityUtil.Fields fields = new EntityUtil.Fields(FIELD_LIST, FIELDS);
EntityReference owner = null;
// Try to find the owner if entity exists
try {
String fqn = getFullyQualifiedName(entity);
entity = getByName(null, fqn, fields);
owner = helper(entity).validateOwnerOrNull();
entity = getByName(null, fqn, FIELDS_OWNER);
return helper(entity).validateOwnerOrNull();
} catch (EntityNotFoundException e) {
// If entity is not found, we can return null for owner and ignore
// this exception
// If entity is not found, we can return null for owner and ignore this exception
}
return owner;
return null;
}
protected EntityReference getOwner(T entity) throws IOException, ParseException {
@ -601,7 +597,7 @@ public abstract class EntityRepository<T> {
}
protected List<TagLabel> getTags(String fqn) {
return !supportsOwner ? null : daoCollection.tagDAO().getTags(fqn);
return !supportsTags ? null : daoCollection.tagDAO().getTags(fqn);
}
protected List<EntityReference> getFollowers(T entity) throws IOException {

258
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java Normal file
View File

@ -0,0 +1,258 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import static org.openmetadata.catalog.Entity.helper;
import java.io.IOException;
import java.net.URI;
import java.text.ParseException;
import java.util.List;
import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.resources.glossary.GlossaryResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.JsonUtils;
public class GlossaryRepository extends EntityRepository<Glossary> {
private static final Fields GLOSSARY_UPDATE_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "owner,tags");
private static final Fields GLOSSARY_PATCH_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "owner,tags");
private final CollectionDAO dao;
public GlossaryRepository(CollectionDAO dao) {
super(
GlossaryResource.COLLECTION_PATH,
Entity.GLOSSARY,
Glossary.class,
dao.glossaryDAO(),
dao,
GLOSSARY_PATCH_FIELDS,
GLOSSARY_UPDATE_FIELDS,
true,
true,
false);
this.dao = dao;
}
@Transaction
public EntityReference getOwnerReference(Glossary glossary) throws IOException {
return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), glossary.getOwner());
}
@Override
public Glossary setFields(Glossary glossary, Fields fields) throws IOException, ParseException {
glossary.setOwner(fields.contains("owner") ? getOwner(glossary) : null);
glossary.setTags(fields.contains("tags") ? getTags(glossary.getName()) : null);
return glossary;
}
@Override
public void prepare(Glossary glossary) throws IOException, ParseException {
glossary.setOwner(helper(glossary).validateOwnerOrNull());
// TODO validate reviewers
glossary.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), glossary.getTags()));
}
@Override
public void storeEntity(Glossary glossary, boolean update) throws IOException {
// Relationships and fields such as href are derived and not stored as part of json
EntityReference owner = glossary.getOwner();
List<TagLabel> tags = glossary.getTags();
// TODO Add relationships for reviewers
// Don't store owner, href and tags as JSON. Build it on the fly based on relationships
glossary.withOwner(null).withHref(null).withTags(null);
if (update) {
dao.glossaryDAO().update(glossary.getId(), JsonUtils.pojoToJson(glossary));
} else {
dao.glossaryDAO().insert(glossary);
}
// Restore the relationships
glossary.withOwner(owner).withTags(tags);
}
@Override
public void storeRelationships(Glossary glossary) {
// TODO Add relationships for related terms, and reviewers
setOwner(glossary, glossary.getOwner());
applyTags(glossary);
}
@Override
public void restorePatchAttributes(Glossary original, Glossary updated) {}
@Override
public EntityInterface<Glossary> getEntityInterface(Glossary entity) {
return new GlossaryEntityInterface(entity);
}
@Override
public EntityUpdater getUpdater(Glossary original, Glossary updated, Operation operation) {
return new GlossaryUpdater(original, updated, operation);
}
public static class GlossaryEntityInterface implements EntityInterface<Glossary> {
private final Glossary entity;
public GlossaryEntityInterface(Glossary entity) {
this.entity = entity;
}
@Override
public UUID getId() {
return entity.getId();
}
@Override
public String getDescription() {
return entity.getDescription();
}
@Override
public String getDisplayName() {
return entity.getDisplayName();
}
@Override
public Boolean isDeleted() {
return entity.getDeleted();
}
@Override
public EntityReference getOwner() {
return entity.getOwner();
}
@Override
public String getFullyQualifiedName() {
return entity.getName();
}
@Override
public List<TagLabel> getTags() {
return entity.getTags();
}
@Override
public Double getVersion() {
return entity.getVersion();
}
@Override
public String getUpdatedBy() {
return entity.getUpdatedBy();
}
@Override
public long getUpdatedAt() {
return entity.getUpdatedAt();
}
@Override
public URI getHref() {
return entity.getHref();
}
@Override
public ChangeDescription getChangeDescription() {
return entity.getChangeDescription();
}
@Override
public EntityReference getEntityReference() {
return new EntityReference()
.withId(getId())
.withName(getFullyQualifiedName())
.withDescription(getDescription())
.withDisplayName(getDisplayName())
.withType(Entity.GLOSSARY);
}
@Override
public Glossary getEntity() {
return entity;
}
@Override
public EntityReference getContainer() {
return null;
}
@Override
public void setId(UUID id) {
entity.setId(id);
}
@Override
public void setDescription(String description) {
entity.setDescription(description);
}
@Override
public void setDisplayName(String displayName) {
entity.setDisplayName(displayName);
}
@Override
public void setUpdateDetails(String updatedBy, long updatedAt) {
entity.setUpdatedBy(updatedBy);
entity.setUpdatedAt(updatedAt);
}
@Override
public void setChangeDescription(Double newVersion, ChangeDescription changeDescription) {
entity.setVersion(newVersion);
entity.setChangeDescription(changeDescription);
}
@Override
public void setOwner(EntityReference owner) {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Glossary withHref(URI href) {
return entity.withHref(href);
}
@Override
public void setTags(List<TagLabel> tags) {
entity.setTags(tags);
}
}
/** Handles entity updated from PUT and POST operation. */
public class GlossaryUpdater extends EntityUpdater {
public GlossaryUpdater(Glossary original, Glossary updated, Operation operation) {
super(original, updated, operation);
}
}
}

300
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java Normal file
View File

@ -0,0 +1,300 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import java.io.IOException;
import java.net.URI;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.GlossaryTerm;
import org.openmetadata.catalog.resources.glossary.GlossaryResource;
import org.openmetadata.catalog.resources.glossary.GlossaryTermResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.Relationship;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.JsonUtils;
@Slf4j
public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
private static final Fields UPDATE_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "tags");
private static final Fields PATCH_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "tags");
private final CollectionDAO dao;
public GlossaryTermRepository(CollectionDAO dao) {
super(
GlossaryTermResource.COLLECTION_PATH,
Entity.GLOSSARY_TERM,
GlossaryTerm.class,
dao.glossaryTermDAO(),
dao,
PATCH_FIELDS,
UPDATE_FIELDS,
true,
false,
false);
this.dao = dao;
}
@Override
public GlossaryTerm setFields(GlossaryTerm entity, Fields fields) throws IOException, ParseException {
entity.setGlossary(getGlossary(entity));
entity.setTags(fields.contains("tags") ? getTags(entity.getFullyQualifiedName()) : null);
return entity;
}
@Override
public void prepare(GlossaryTerm entity) throws IOException, ParseException {
// Set fully qualified name
if (entity.getParent() == null) {
entity.setFullyQualifiedName(entity.getGlossary().getName() + "." + entity.getName());
} else {
entity.setFullyQualifiedName(entity.getParent().getName() + "." + entity.getName());
}
// Validate related terms
List<EntityReference> validatedRelatedTerms = new ArrayList<>();
for (EntityReference related : entity.getRelatedTerms()) {
validatedRelatedTerms.add(daoCollection.glossaryTermDAO().findEntityReferenceById(related.getId()));
}
entity.setRelatedTerms(validatedRelatedTerms);
// Set tags
entity.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), entity.getTags()));
}
@Override
public void storeEntity(GlossaryTerm entity, boolean update) throws IOException {
// Relationships and fields such as href are derived and not stored as part of json
List<TagLabel> tags = entity.getTags();
// TODO Add relationships for reviewers
EntityReference glossary = entity.getGlossary();
EntityReference parentTerm = entity.getParent();
// Don't store owner, dashboard, href and tags as JSON. Build it on the fly based on relationships
entity.withGlossary(null).withParent(null).withHref(null).withTags(null);
if (update) {
dao.glossaryTermDAO().update(entity.getId(), JsonUtils.pojoToJson(entity));
} else {
dao.glossaryTermDAO().insert(entity);
}
// Restore the relationships
entity.withGlossary(glossary).withParent(parentTerm).withTags(tags);
}
@Override
public void storeRelationships(GlossaryTerm entity) {
// TODO Add relationships for related terms, and reviewers
daoCollection
.relationshipDAO()
.insert(
entity.getGlossary().getId(),
entity.getId(),
Entity.GLOSSARY,
Entity.GLOSSARY_TERM,
Relationship.CONTAINS.ordinal());
if (entity.getParent() != null) {
daoCollection
.relationshipDAO()
.insert(
entity.getParent().getId(),
entity.getId(),
Entity.GLOSSARY,
Entity.GLOSSARY_TERM,
Relationship.CONTAINS.ordinal());
}
applyTags(entity);
}
@Override
public void restorePatchAttributes(GlossaryTerm original, GlossaryTerm updated) {}
protected EntityReference getGlossary(GlossaryTerm term) throws IOException {
List<String> refs =
daoCollection
.relationshipDAO()
.findFrom(
term.getId().toString(), Entity.GLOSSARY_TERM, Relationship.CONTAINS.ordinal(), Entity.GLOSSARY, null);
if (refs.size() != 1) {
LOG.warn(
"Possible database issues - multiple owners found for entity {} with type {}", term.getId(), Entity.GLOSSARY);
}
return daoCollection.glossaryDAO().findEntityReferenceById(UUID.fromString(refs.get(0)));
}
@Override
public EntityInterface<GlossaryTerm> getEntityInterface(GlossaryTerm entity) {
return new GlossaryTermEntityInterface(entity);
}
@Override
public EntityUpdater getUpdater(GlossaryTerm original, GlossaryTerm updated, Operation operation) {
return new GlossaryTermUpdater(original, updated, operation);
}
public static class GlossaryTermEntityInterface implements EntityInterface<GlossaryTerm> {
private final GlossaryTerm entity;
public GlossaryTermEntityInterface(GlossaryTerm entity) {
this.entity = entity;
}
@Override
public UUID getId() {
return entity.getId();
}
@Override
public String getDescription() {
return entity.getDescription();
}
@Override
public String getDisplayName() {
return entity.getDisplayName();
}
@Override
public Boolean isDeleted() {
return entity.getDeleted();
}
@Override
public EntityReference getOwner() {
return null;
}
@Override
public String getFullyQualifiedName() {
return entity.getFullyQualifiedName();
}
@Override
public List<TagLabel> getTags() {
return entity.getTags();
}
@Override
public Double getVersion() {
return entity.getVersion();
}
@Override
public String getUpdatedBy() {
return entity.getUpdatedBy();
}
@Override
public long getUpdatedAt() {
return entity.getUpdatedAt();
}
@Override
public URI getHref() {
return entity.getHref();
}
@Override
public ChangeDescription getChangeDescription() {
return entity.getChangeDescription();
}
@Override
public EntityReference getEntityReference() {
return new EntityReference()
.withId(getId())
.withName(getFullyQualifiedName())
.withDescription(getDescription())
.withDisplayName(getDisplayName())
.withType(Entity.GLOSSARY_TERM);
}
@Override
public GlossaryTerm getEntity() {
return entity;
}
@Override
public EntityReference getContainer() {
return null;
}
@Override
public void setId(UUID id) {
entity.setId(id);
}
@Override
public void setDescription(String description) {
entity.setDescription(description);
}
@Override
public void setDisplayName(String displayName) {
entity.setDisplayName(displayName);
}
@Override
public void setUpdateDetails(String updatedBy, long updatedAt) {
entity.setUpdatedBy(updatedBy);
entity.setUpdatedAt(updatedAt);
}
@Override
public void setChangeDescription(Double newVersion, ChangeDescription changeDescription) {
entity.setVersion(newVersion);
entity.setChangeDescription(changeDescription);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public GlossaryTerm withHref(URI href) {
return entity.withHref(href);
}
@Override
public void setTags(List<TagLabel> tags) {
entity.setTags(tags);
}
}
/** Handles entity updated from PUT and POST operation. */
public class GlossaryTermUpdater extends EntityUpdater {
public GlossaryTermUpdater(GlossaryTerm original, GlossaryTerm updated, Operation operation) {
super(original, updated, operation);
}
@Override
public void entitySpecificUpdate() throws IOException {
// TODO
}
}
}

397
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java Normal file
View File

@ -0,0 +1,397 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.resources.glossary;
import com.google.inject.Inject;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.PATCH;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateGlossary;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.jdbi3.CollectionDAO;
import org.openmetadata.catalog.jdbi3.GlossaryRepository;
import org.openmetadata.catalog.resources.Collection;
import org.openmetadata.catalog.security.Authorizer;
import org.openmetadata.catalog.security.SecurityUtil;
import org.openmetadata.catalog.type.EntityHistory;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.Include;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.RestUtil;
import org.openmetadata.catalog.util.RestUtil.DeleteResponse;
import org.openmetadata.catalog.util.RestUtil.PatchResponse;
import org.openmetadata.catalog.util.RestUtil.PutResponse;
import org.openmetadata.catalog.util.ResultList;
@Path("/v1/glossaries")
@Api(value = "Glossary collection", tags = "Glossary collection")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "glossaries")
public class GlossaryResource {
public static final String COLLECTION_PATH = "v1/glossaries/";
private final GlossaryRepository dao;
private final Authorizer authorizer;
public static void addHref(UriInfo uriInfo, EntityReference ref) {
ref.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, ref.getId()));
}
public static List<Glossary> addHref(UriInfo uriInfo, List<Glossary> glossaries) {
Optional.ofNullable(glossaries).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
return glossaries;
}
public static Glossary addHref(UriInfo uriInfo, Glossary glossary) {
glossary.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, glossary.getId()));
Entity.withHref(uriInfo, glossary.getOwner());
return glossary;
}
@Inject
public GlossaryResource(CollectionDAO dao, Authorizer authorizer) {
Objects.requireNonNull(dao, "GlossaryRepository must not be null");
this.dao = new GlossaryRepository(dao);
this.authorizer = authorizer;
}
public static class GlossaryList extends ResultList<Glossary> {
@SuppressWarnings("unused")
GlossaryList() {
// Empty constructor needed for deserialization
}
public GlossaryList(List<Glossary> data, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(data, beforeCursor, afterCursor, total);
}
}
static final String FIELDS = "owner,tags";
public static final List<String> FIELD_LIST = Arrays.asList(FIELDS.replaceAll(" ", "").split(","));
@GET
@Valid
@Operation(
summary = "List Glossaries",
tags = "glossaries",
description =
"Get a list of glossaries. Use `fields` parameter to get only necessary fields. "
+ " Use cursor-based pagination to limit the number "
+ "entries in the list using `limit` and `before` or `after` query params.",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of glossaries",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryList.class)))
})
public ResultList<Glossary> list(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(description = "Limit the number glossaries returned. (1 to 1000000, " + "default = 10)")
@DefaultValue("10")
@Min(1)
@Max(1000000)
@QueryParam("limit")
int limitParam,
@Parameter(description = "Returns list of glossaries before this cursor", schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(description = "Returns list of glossaries after this cursor", schema = @Schema(type = "string"))
@QueryParam("after")
String after,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException, GeneralSecurityException, ParseException {
RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam);
ResultList<Glossary> glossary;
if (before != null) { // Reverse paging
glossary = dao.listBefore(uriInfo, fields, null, limitParam, before, include); // Ask for one extra entry
} else { // Forward paging or first page
glossary = dao.listAfter(uriInfo, fields, null, limitParam, after, include);
}
addHref(uriInfo, glossary.getData());
return glossary;
}
@GET
@Path("/{id}")
@Operation(
summary = "Get a glossary",
tags = "glossaries",
description = "Get a glossary by `id`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The glossary",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
@ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found")
})
public Glossary get(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") String id,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException, ParseException {
Fields fields = new Fields(FIELD_LIST, fieldsParam);
return addHref(uriInfo, dao.get(uriInfo, id, fields, include));
}
@GET
@Path("/name/{name}")
@Operation(
summary = "Get a glossary by name",
tags = "glossaries",
description = "Get a glossary by name.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The glossary",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
@ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found")
})
public Glossary getByName(
@Context UriInfo uriInfo,
@PathParam("name") String name,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException, ParseException {
Fields fields = new Fields(FIELD_LIST, fieldsParam);
Glossary glossary = dao.getByName(uriInfo, name, fields, include);
return addHref(uriInfo, glossary);
}
@GET
@Path("/{id}/versions")
@Operation(
summary = "List glossary versions",
tags = "glossaries",
description = "Get a list of all the versions of a glossary identified by `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of glossary versions",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class)))
})
public EntityHistory listVersions(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id)
throws IOException, ParseException {
return dao.listVersions(id);
}
@GET
@Path("/{id}/versions/{version}")
@Operation(
summary = "Get a version of the glossaries",
tags = "glossaries",
description = "Get a version of the glossary by given `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "glossaries",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
@ApiResponse(
responseCode = "404",
description = "Glossary for instance {id} and version {version} is " + "not found")
})
public Glossary getVersion(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id,
@Parameter(
description = "glossary version number in the form `major`.`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version")
String version)
throws IOException, ParseException {
return dao.getVersion(id, version);
}
@POST
@Operation(
summary = "Create a glossary",
tags = "glossaries",
description = "Create a new glossary.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The glossary",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = CreateGlossary.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response create(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create)
throws IOException, ParseException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
Glossary glossary = getGlossary(securityContext, create);
glossary = addHref(uriInfo, dao.create(uriInfo, glossary));
return Response.created(glossary.getHref()).entity(glossary).build();
}
@PATCH
@Path("/{id}")
@Operation(
summary = "Update a glossary",
tags = "glossaries",
description = "Update an existing glossary using JsonPatch.",
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
public Response updateDescription(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") String id,
@RequestBody(
description = "JsonPatch with array of operations",
content =
@Content(
mediaType = MediaType.APPLICATION_JSON_PATCH_JSON,
examples = {
@ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]")
}))
JsonPatch patch)
throws IOException, ParseException {
Fields fields = new Fields(FIELD_LIST, FIELDS);
Glossary glossary = dao.get(uriInfo, id, fields);
SecurityUtil.checkAdminRoleOrPermissions(
authorizer, securityContext, dao.getEntityInterface(glossary).getEntityReference(), patch);
PatchResponse<Glossary> response =
dao.patch(uriInfo, UUID.fromString(id), securityContext.getUserPrincipal().getName(), patch);
addHref(uriInfo, response.getEntity());
return response.toResponse();
}
@PUT
@Operation(
summary = "Create or update a glossary",
tags = "glossaries",
description = "Create a new glossary, if it does not exist or update an existing glossary.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The glossary",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response createOrUpdate(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create)
throws IOException, ParseException {
Glossary glossary = getGlossary(securityContext, create);
EntityReference owner = dao.getOriginalOwner(glossary);
SecurityUtil.checkAdminRoleOrPermissions(authorizer, securityContext, owner);
PutResponse<Glossary> response = dao.createOrUpdate(uriInfo, glossary);
addHref(uriInfo, response.getEntity());
return response.toResponse();
}
@DELETE
@Path("/{id}")
@Operation(
summary = "Delete a Glossary",
tags = "glossaries",
description = "Delete a glossary by `id`.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "glossary for instance {id} is not found")
})
public Response delete(@Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("id") String id)
throws IOException, ParseException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
DeleteResponse<Glossary> response = dao.delete(securityContext.getUserPrincipal().getName(), id);
return response.toResponse();
}
private Glossary getGlossary(SecurityContext securityContext, CreateGlossary create) {
return new Glossary()
.withId(UUID.randomUUID())
.withName(create.getName())
.withDisplayName(create.getDisplayName())
.withDescription(create.getDescription())
.withReviewers(create.getReviewers())
.withTags(create.getTags())
.withOwner(create.getOwner())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(System.currentTimeMillis());
}
}

401
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResource.java Normal file
View File

@ -0,0 +1,401 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.resources.glossary;
import com.google.inject.Inject;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.PATCH;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateGlossaryTerm;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.entity.data.GlossaryTerm;
import org.openmetadata.catalog.jdbi3.CollectionDAO;
import org.openmetadata.catalog.jdbi3.GlossaryTermRepository;
import org.openmetadata.catalog.resources.Collection;
import org.openmetadata.catalog.security.Authorizer;
import org.openmetadata.catalog.security.SecurityUtil;
import org.openmetadata.catalog.type.EntityHistory;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.Include;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.RestUtil;
import org.openmetadata.catalog.util.RestUtil.DeleteResponse;
import org.openmetadata.catalog.util.RestUtil.PatchResponse;
import org.openmetadata.catalog.util.RestUtil.PutResponse;
import org.openmetadata.catalog.util.ResultList;
@Path("/v1/glossaryTerms")
@Api(value = "Glossary collection", tags = "Glossary collection")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "glossaries")
public class GlossaryTermResource {
public static final String COLLECTION_PATH = "v1/glossaryTerms/";
private final GlossaryTermRepository dao;
private final Authorizer authorizer;
public static List<GlossaryTerm> addHref(UriInfo uriInfo, List<GlossaryTerm> terms) {
Optional.ofNullable(terms).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
return terms;
}
public static GlossaryTerm addHref(UriInfo uriInfo, GlossaryTerm term) {
term.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, term.getId()));
Entity.withHref(uriInfo, term.getGlossary());
return term;
}
@Inject
public GlossaryTermResource(CollectionDAO dao, Authorizer authorizer) {
Objects.requireNonNull(dao, "GlossaryTermRepository must not be null");
this.dao = new GlossaryTermRepository(dao);
this.authorizer = authorizer;
}
public static class GlossaryTermList extends ResultList<GlossaryTerm> {
@SuppressWarnings("unused")
GlossaryTermList() {
// Empty constructor needed for deserialization
}
public GlossaryTermList(List<GlossaryTerm> data, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(data, beforeCursor, afterCursor, total);
}
}
static final String FIELDS = "tags";
public static final List<String> FIELD_LIST = Arrays.asList(FIELDS.replaceAll(" ", "").split(","));
@GET
@Valid
@Operation(
summary = "List glossary terms",
tags = "glossaries",
description =
"Get a list of glossary terms. Use `fields` parameter to get only necessary fields. "
+ " Use cursor-based pagination to limit the number "
+ "entries in the list using `limit` and `before` or `after` query params.",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of glossary terms",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryTermList.class)))
})
public ResultList<GlossaryTerm> list(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(description = "Limit the number glossary terms returned. (1 to 1000000, " + "default = 10)")
@DefaultValue("10")
@Min(1)
@Max(1000000)
@QueryParam("limit")
int limitParam,
@Parameter(description = "Returns list of glossary terms before this cursor", schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(description = "Returns list of glossary terms after this cursor", schema = @Schema(type = "string"))
@QueryParam("after")
String after,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException, GeneralSecurityException, ParseException {
RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam);
ResultList<GlossaryTerm> terms;
if (before != null) { // Reverse paging
terms = dao.listBefore(uriInfo, fields, null, limitParam, before, include); // Ask for one extra entry
} else { // Forward paging or first page
terms = dao.listAfter(uriInfo, fields, null, limitParam, after, include);
}
addHref(uriInfo, terms.getData());
return terms;
}
@GET
@Path("/{id}")
@Operation(
summary = "Get a glossary",
tags = "glossaries",
description = "Get a glossary by `id`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The glossary",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
@ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found")
})
public GlossaryTerm get(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") String id,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException, ParseException {
Fields fields = new Fields(FIELD_LIST, fieldsParam);
return addHref(uriInfo, dao.get(uriInfo, id, fields, include));
}
@GET
@Path("/name/{name}")
@Operation(
summary = "Get a glossary by name",
tags = "glossaries",
description = "Get a glossary by name.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The glossary",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
@ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found")
})
public GlossaryTerm getByName(
@Context UriInfo uriInfo,
@PathParam("name") String name,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException, ParseException {
Fields fields = new Fields(FIELD_LIST, fieldsParam);
GlossaryTerm term = dao.getByName(uriInfo, name, fields, include);
return addHref(uriInfo, term);
}
@GET
@Path("/{id}/versions")
@Operation(
summary = "List glossary versions",
tags = "glossaries",
description = "Get a list of all the versions of a glossary identified by `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of glossary versions",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class)))
})
public EntityHistory listVersions(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id)
throws IOException, ParseException {
return dao.listVersions(id);
}
@GET
@Path("/{id}/versions/{version}")
@Operation(
summary = "Get a version of the glossaries",
tags = "glossaries",
description = "Get a version of the glossary by given `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "glossaries",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
@ApiResponse(
responseCode = "404",
description = "Glossary for instance {id} and version {version} is " + "not found")
})
public GlossaryTerm getVersion(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id,
@Parameter(
description = "glossary version number in the form `major`.`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version")
String version)
throws IOException, ParseException {
return dao.getVersion(id, version);
}
@POST
@Operation(
summary = "Create a glossary",
tags = "glossaries",
description = "Create a new glossary.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The glossary",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryTerm.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response create(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossaryTerm create)
throws IOException, ParseException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
GlossaryTerm term = getGlossaryTerm(securityContext, create);
term = addHref(uriInfo, dao.create(uriInfo, term));
return Response.created(term.getHref()).entity(term).build();
}
@PATCH
@Path("/{id}")
@Operation(
summary = "Update a glossary",
tags = "glossaries",
description = "Update an existing glossary using JsonPatch.",
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
public Response updateDescription(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") String id,
@RequestBody(
description = "JsonPatch with array of operations",
content =
@Content(
mediaType = MediaType.APPLICATION_JSON_PATCH_JSON,
examples = {
@ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]")
}))
JsonPatch patch)
throws IOException, ParseException {
Fields fields = new Fields(FIELD_LIST, FIELDS);
GlossaryTerm term = dao.get(uriInfo, id, fields);
SecurityUtil.checkAdminRoleOrPermissions(
authorizer, securityContext, dao.getEntityInterface(term).getEntityReference(), patch);
PatchResponse<GlossaryTerm> response =
dao.patch(uriInfo, UUID.fromString(id), securityContext.getUserPrincipal().getName(), patch);
addHref(uriInfo, response.getEntity());
return response.toResponse();
}
@PUT
@Operation(
summary = "Create or update a glossary",
tags = "glossaries",
description = "Create a new glossary, if it does not exist or update an existing glossary.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The glossary",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryTerm.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response createOrUpdate(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossaryTerm create)
throws IOException, ParseException {
GlossaryTerm term = getGlossaryTerm(securityContext, create);
EntityReference owner = dao.getOriginalOwner(term);
SecurityUtil.checkAdminRoleOrPermissions(authorizer, securityContext, owner);
PutResponse<GlossaryTerm> response = dao.createOrUpdate(uriInfo, term);
addHref(uriInfo, response.getEntity());
return response.toResponse();
}
@DELETE
@Path("/{id}")
@Operation(
summary = "Delete a Glossary",
tags = "glossaries",
description = "Delete a glossary by `id`.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "glossary for instance {id} is not found")
})
public Response delete(@Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("id") String id)
throws IOException, ParseException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
DeleteResponse<GlossaryTerm> response = dao.delete(securityContext.getUserPrincipal().getName(), id);
return response.toResponse();
}
private GlossaryTerm getGlossaryTerm(SecurityContext securityContext, CreateGlossaryTerm create) throws IOException {
EntityReference glossary = Entity.getEntityReference(Entity.GLOSSARY, create.getGlossaryId());
EntityReference parentTerm =
create.getParentId() != null ? Entity.getEntityReference(Entity.GLOSSARY_TERM, create.getParentId()) : null;
return new GlossaryTerm()
.withId(UUID.randomUUID())
.withName(create.getName())
.withDisplayName(create.getDisplayName())
.withDescription(create.getDescription())
.withSynonyms(create.getSynonyms())
.withGlossary(glossary)
.withParent(parentTerm)
.withRelatedTerms(create.getRelatedTerms())
.withReferences(create.getReferences())
.withReviewers(create.getReviewers())
.withTags(create.getTags())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(System.currentTimeMillis());
}
}

26
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java
View File

@ -153,6 +153,9 @@ public class SearchResource {
case "team_search_index":
searchSourceBuilder = buildTeamSearchBuilder(query, from, size);
break;
case "glossary_search_index":
searchSourceBuilder = buildGlossarySearchBuilder(query, from, size);
break;
default:
searchSourceBuilder = buildAggregateSearchBuilder(query, from, size);
break;
@ -380,4 +383,27 @@ public class SearchResource {
return searchSourceBuilder;
}
private SearchSourceBuilder buildGlossarySearchBuilder(String query, int from, int size) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
HighlightBuilder.Field highlightGlossaryName = new HighlightBuilder.Field("glossary_name");
highlightGlossaryName.highlighterType("unified");
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field("description");
highlightDescription.highlighterType("unified");
HighlightBuilder hb = new HighlightBuilder();
hb.field(highlightDescription);
hb.field(highlightGlossaryName);
hb.preTags("<span class=\"text-highlighter\">");
hb.postTags("</span>");
searchSourceBuilder
.query(QueryBuilders.queryStringQuery(query).field("glossary_name", 5.0f).field("description").lenient(true))
.aggregation(AggregationBuilders.terms("EntityType").field("entity_type"))
.aggregation(AggregationBuilders.terms("Tier").field("tier"))
.aggregation(AggregationBuilders.terms("Tags").field("tags"))
.highlighter(hb)
.from(from)
.size(size);
return searchSourceBuilder;
}
}

41
catalog-rest-service/src/main/resources/json/schema/api/data/createGlossary.json Normal file
View File

@ -0,0 +1,41 @@
{
"$id": "https://open-metadata.org/schema/api/data/createGlossary.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Create Glossary entity request",
"description": "Create Glossary entity request",
"type": "object",
"properties" : {
"name": {
"description": "Name that identifies this glossary.",
"$ref": "../../entity/data/glossary.json#/definitions/name"
},
"displayName": {
"description": "Display Name that identifies this glossary.",
"type": "string"
},
"description": {
"description": "Description of the glossary instance.",
"type": "string"
},
"reviewers": {
"description": "User names of the reviewers for this glossary.",
"type": "array",
"items" : {
"type": "string"
}
},
"owner": {
"description": "Owner of this glossary",
"$ref": "../../type/entityReference.json"
},
"tags": {
"description": "Tags for this glossary",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
}
},
"required": ["name"]
}

73
catalog-rest-service/src/main/resources/json/schema/api/data/createGlossaryTerm.json Normal file
View File

@ -0,0 +1,73 @@
{
"$id": "https://open-metadata.org/schema/api/data/createGlossary.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Create Glossary entity request",
"description": "Create Glossary entity request",
"type": "object",
"properties": {
"glossaryId" : {
"description": "UUID of the glossary.",
"$ref" : "../../type/basic.json#/definitions/uuid"
},
"parentId" : {
"description": "UUID of the parent term. When null, the term is at the root of the glossary.",
"$ref" : "../../type/basic.json#/definitions/uuid"
},
"name": {
"description": "Preferred name for the glossary term.",
"$ref": "../../entity/data/glossaryTerm.json#/definitions/name"
},
"displayName": {
"description": "Display Name that identifies this glossary.",
"type": "string"
},
"description": {
"description": "Description of the glossary term.",
"type": "string"
},
"synonyms": {
"description": "Alternate names that are synonyms or near-synonyms for the glossary term.",
"type": "array",
"items": {
"$ref": "../../entity/data/glossaryTerm.json#/definitions/name"
}
},
"children": {
"description": "Other glossary terms that are children of this glossary term.",
"type": "array",
"items": {
"$ref": "../../type/entityReference.json"
}
},
"relatedTerms": {
"description": "Other glossary terms that are related to this glossary term.",
"type": "array",
"items": {
"$ref": "../../type/entityReference.json"
}
},
"references": {
"description": "Link to a reference from an external glossary.",
"$ref": "../../entity/data/glossaryTerm.json#/definitions/termReference"
},
"reviewers": {
"description": "User names of the reviewers for this glossary.",
"type": "array",
"items": {
"type": "string"
}
},
"tags": {
"description": "Tags for this glossary term.",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
}
},
"required": [
"glossaryId",
"name"
]
}

82
catalog-rest-service/src/main/resources/json/schema/entity/data/glossary.json Normal file
View File

@ -0,0 +1,82 @@
{
"$id": "https://open-metadata.org/schema/entity/data/glossary.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Glossary",
"description": "This schema defines the Glossary entity based on SKOS.",
"type": "object",
"definitions": {
"name": {
"description": "Name that identifies a glossary term.",
"type": "string",
"minLength": 1,
"maxLength": 128
}
},
"properties": {
"id": {
"description": "Unique identifier of a glossary instance.",
"$ref": "../../type/basic.json#/definitions/uuid"
},
"name": {
"description": "Preferred name for the glossary term.",
"type": "string",
"$ref": "#/definitions/name"
},
"displayName": {
"description": "Display Name that identifies this glossary.",
"type": "string"
},
"description": {
"description": "Description of the glossary.",
"type": "string"
},
"version": {
"description": "Metadata version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/entityVersion"
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"updatedBy": {
"description": "User who made the update.",
"type": "string"
},
"href": {
"description": "Link to the resource corresponding to this entity.",
"$ref": "../../type/basic.json#/definitions/href"
},
"reviewers": {
"description": "User names of the reviewers for this glossary.",
"type": "array",
"items": {
"type": "string"
}
},
"owner": {
"description": "Owner of this glossary.",
"$ref": "../../type/entityReference.json"
},
"tags": {
"description": "Tags for this glossary.",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"changeDescription": {
"description": "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
}
},
"required": [
"id",
"name"
]
}

129
catalog-rest-service/src/main/resources/json/schema/entity/data/glossaryTerm.json Normal file
View File

@ -0,0 +1,129 @@
{
"$id": "https://open-metadata.org/schema/entity/data/glossaryTerm.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "GlossaryTerm",
"description": "This schema defines the Glossary term entities.",
"type": "object",
"definitions": {
"name": {
"description": "Name that identifies a glossary term.",
"type": "string",
"minLength": 1,
"maxLength": 128
},
"termReference": {
"type": "object",
"properties": {
"name": {
"description": "Name that identifies the source of an external glossary term. Example `HealthCare.gov`",
"type": "string"
},
"endpoint": {
"description": "Name that identifies the source of an external glossary term. Example `HealthCare.gov`",
"type": "string",
"format": "uri"
}
}
}
},
"properties": {
"id": {
"description": "Unique identifier of a glossary term instance.",
"$ref": "../../type/basic.json#/definitions/uuid"
},
"name": {
"description": "Preferred name for the glossary term.",
"$ref": "#/definitions/name"
},
"displayName": {
"description": "Display Name that identifies this glossary.",
"type": "string"
},
"description": {
"description": "Description of the glossary term.",
"type": "string"
},
"fullyQualifiedName": {
"description": "A unique name that identifies a glossary term. It captures name hierarchy of glossary of terms in the form of `glossaryName.parentTerm.childTerm`.",
"type": "string",
"minLength": 1,
"maxLength": 256
},
"synonyms": {
"description": "Alternate names that are synonyms or near-synonyms for the glossary term.",
"type": "array",
"items": {
"$ref": "#/definitions/name"
}
},
"glossary": {
"description": "Glosary that this term belongs to.",
"$ref" : "../../type/entityReference.json"
},
"parent": {
"description": "Parent glossary term that this term is child of. When `null` this term is the root term of the glossary.",
"$ref" : "../../type/entityReference.json"
},
"children": {
"description": "Other glossary terms that are children of this glossary term.",
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList"
},
"relatedTerms": {
"description": "Other glossary terms that are related to this glossary term.",
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList"
},
"references": {
"description": "Link to a reference from an external glossary.",
"$ref": "#/definitions/termReference"
},
"version": {
"description": "Metadata version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/entityVersion"
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"updatedBy": {
"description": "User who made the update.",
"type": "string"
},
"skos": {
"description": "SKOS data in JSON-LD format",
"type": "string"
},
"href": {
"description": "Link to the resource corresponding to this entity.",
"$ref": "../../type/basic.json#/definitions/href"
},
"reviewers": {
"description": "User names of the reviewers for this glossary.",
"type": "array",
"items": {
"type": "string"
}
},
"tags": {
"description": "Tags for this glossary term.",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"changeDescription": {
"description": "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
}
},
"required": [
"id",
"name",
"glossary"
]
}

12
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java
View File

@ -829,15 +829,13 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
// Create an entity with owner
K request = createRequest(getEntityName(test), "description", "displayName", USER_OWNER1);
createAndCheckEntity(request, ADMIN_AUTH_HEADERS);
createEntity(request, ADMIN_AUTH_HEADERS);
// Update description and remove owner as non-owner
// Expect to throw an exception since only owner or admin can update resource
K updateRequest = createRequest(getEntityName(test), "newdescription", "displayName", null);
K updateRequest = createRequest(getEntityName(test), "newDescription", "displayName", null);
HttpResponseException exception =
assertThrows(
HttpResponseException.class,
() -> updateAndCheckEntity(updateRequest, OK, TEST_AUTH_HEADERS, UpdateType.NO_CHANGE, null));
assertThrows(HttpResponseException.class, () -> updateEntity(updateRequest, OK, TEST_AUTH_HEADERS));
TestUtils.assertResponse(
exception, FORBIDDEN, "Principal: CatalogPrincipal{name='test'} " + "does not have permissions");
}
@ -846,7 +844,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
void put_entityNullDescriptionUpdate_200(TestInfo test) throws IOException {
// Create entity with null description
K request = createRequest(getEntityName(test), null, "displayName", null);
T entity = createAndCheckEntity(request, ADMIN_AUTH_HEADERS);
T entity = createEntity(request, ADMIN_AUTH_HEADERS);
EntityInterface<T> entityInterface = getEntityInterface(entity);
// Update null description with a new description
@ -861,7 +859,7 @@ public abstract class EntityResourceTest<T, K> extends CatalogApplicationTest {
void put_entityEmptyDescriptionUpdate_200(TestInfo test) throws IOException {
// Create entity with empty description
K request = createRequest(getEntityName(test), "", "displayName", null);
T entity = createAndCheckEntity(request, ADMIN_AUTH_HEADERS);
T entity = createEntity(request, ADMIN_AUTH_HEADERS);
EntityInterface<T> entityInterface = getEntityInterface(entity);
// Update empty description with a new description

117
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryResourceTest.java Normal file
View File

@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.resources.glossary;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Map;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.TestMethodOrder;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateGlossary;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface;
import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.TestUtils;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlossary> {
public GlossaryResourceTest() {
super(
Entity.GLOSSARY,
Glossary.class,
GlossaryResource.GlossaryList.class,
"glossaries",
GlossaryResource.FIELDS,
false,
true,
true,
true);
}
@BeforeAll
public void setup(TestInfo test) throws IOException, URISyntaxException {
super.setup(test);
}
@Override
public CreateGlossary createRequest(String name, String description, String displayName, EntityReference owner) {
return new CreateGlossary()
.withName(name)
.withDescription(description)
.withDisplayName(displayName)
.withOwner(owner);
}
@Override
public EntityReference getContainer(CreateGlossary createRequest) {
return null;
}
@Override
public void validateCreatedEntity(
Glossary createdEntity, CreateGlossary createRequest, Map<String, String> authHeaders)
throws HttpResponseException {
validateCommonEntityFields(
getEntityInterface(createdEntity),
createRequest.getDescription(),
TestUtils.getPrincipal(authHeaders),
createRequest.getOwner());
// Entity specific validation
TestUtils.validateTags(createRequest.getTags(), createdEntity.getTags());
}
@Override
public void validateUpdatedEntity(Glossary updated, CreateGlossary request, Map<String, String> authHeaders)
throws HttpResponseException {
validateCreatedEntity(updated, request, authHeaders);
}
@Override
public void compareEntities(Glossary expected, Glossary patched, Map<String, String> authHeaders)
throws HttpResponseException {
validateCommonEntityFields(
getEntityInterface(patched),
expected.getDescription(),
TestUtils.getPrincipal(authHeaders),
expected.getOwner());
// Entity specific validation
TestUtils.validateTags(expected.getTags(), patched.getTags());
}
@Override
public GlossaryEntityInterface getEntityInterface(Glossary entity) {
return new GlossaryEntityInterface(entity);
}
@Override
public void validateGetWithDifferentFields(Glossary entity, boolean byName) throws HttpResponseException {}
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
if (expected == actual) {
return;
}
assertCommonFieldChange(fieldName, expected, actual);
}
}

137
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryTermResourceTest.java Normal file
View File

@ -0,0 +1,137 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.resources.glossary;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.catalog.util.TestUtils.validateEntityReference;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Map;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.TestMethodOrder;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateGlossary;
import org.openmetadata.catalog.api.data.CreateGlossaryTerm;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.entity.data.GlossaryTerm;
import org.openmetadata.catalog.jdbi3.GlossaryTermRepository.GlossaryTermEntityInterface;
import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.TestUtils;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, CreateGlossaryTerm> {
public static Glossary GLOSSARY;
public GlossaryTermResourceTest() {
super(
Entity.GLOSSARY_TERM,
GlossaryTerm.class,
GlossaryTermResource.GlossaryTermList.class,
"glossaryTerms",
GlossaryTermResource.FIELDS,
false,
false,
true,
false);
}
@BeforeAll
public void setup(TestInfo test) throws IOException, URISyntaxException {
super.setup(test);
GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest();
CreateGlossary createGlossary = glossaryResourceTest.createRequest(test);
GLOSSARY = glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);
}
@Override
public CreateGlossaryTerm createRequest(String name, String description, String displayName, EntityReference owner) {
return new CreateGlossaryTerm()
.withName(name)
.withDescription(description)
.withDisplayName(displayName)
.withGlossaryId(GLOSSARY.getId());
}
@Override
public EntityReference getContainer(CreateGlossaryTerm createRequest) {
return null;
}
/**
* A method variant to be called form other tests to create a glossary without depending on Database, DatabaseService
* set up in the {@code setup()} method
*/
public GlossaryTerm createEntity(TestInfo test, int index) throws IOException {
CreateGlossaryTerm create = new CreateGlossaryTerm().withName(getEntityName(test, index));
return createEntity(create, ADMIN_AUTH_HEADERS);
}
@Override
public void validateCreatedEntity(
GlossaryTerm createdEntity, CreateGlossaryTerm createRequest, Map<String, String> authHeaders)
throws HttpResponseException {
validateCommonEntityFields(
getEntityInterface(createdEntity), createRequest.getDescription(), TestUtils.getPrincipal(authHeaders), null);
validateEntityReference(createdEntity.getGlossary());
assertEquals(createRequest.getGlossaryId(), createdEntity.getGlossary().getId());
// Entity specific validation
TestUtils.validateTags(createRequest.getTags(), createdEntity.getTags());
}
@Override
public void validateUpdatedEntity(GlossaryTerm updated, CreateGlossaryTerm request, Map<String, String> authHeaders)
throws HttpResponseException {
validateCreatedEntity(updated, request, authHeaders);
}
@Override
public void compareEntities(GlossaryTerm expected, GlossaryTerm patched, Map<String, String> authHeaders)
throws HttpResponseException {
validateCommonEntityFields(
getEntityInterface(patched), expected.getDescription(), TestUtils.getPrincipal(authHeaders), null);
validateEntityReference(patched.getGlossary());
assertEquals(expected.getGlossary().getId(), patched.getGlossary().getId());
// Entity specific validation
TestUtils.validateTags(expected.getTags(), patched.getTags());
}
@Override
public GlossaryTermEntityInterface getEntityInterface(GlossaryTerm entity) {
return new GlossaryTermEntityInterface(entity);
}
@Override
public void validateGetWithDifferentFields(GlossaryTerm entity, boolean byName) throws HttpResponseException {}
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
if (expected == actual) {
return;
}
assertCommonFieldChange(fieldName, expected, actual);
}
}

1332
ingestion/examples/sample_data/thesauruses/hobbies-SKOS.json Normal file
View File

File diff suppressed because it is too large Load Diff

2
ingestion/pipelines/metadata_to_es.json
View File

@ -5,6 +5,7 @@
"include_tables": "true",
"include_topics": "true",
"include_dashboards": "true",
"include_glossary": "true",
"limit_records": 10
}
},
@ -14,6 +15,7 @@
"index_tables": "true",
"index_topics": "true",
"index_dashboards": "true",
"index_glossary": "true",
"es_host": "localhost",
"es_port": 9200
}

17
ingestion/src/metadata/ingestion/models/table_metadata.py
View File

@ -104,6 +104,23 @@ class TopicESDocument(BaseModel):
doc_as_upsert: bool = True
class GlossaryESDocument(BaseModel):
"""Glossary Elastic Search Mapping doc"""
glossary_id: str
glossary_name: str
entity_type: str = "glossary"
suggest: List[dict]
description: Optional[str] = None
last_updated_timestamp: Optional[int]
tags: List[str]
fqdn: str
tier: Optional[str] = None
schema_description: Optional[str] = None
owner: str
followers: List[str]
class DashboardESDocument(BaseModel):
"""Elastic Search Mapping doc for Dashboards"""

6
ingestion/src/metadata/ingestion/ometa/ometa_api.py
View File

@ -25,6 +25,7 @@ from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
from metadata.generated.schema.entity.data.chart import Chart
from metadata.generated.schema.entity.data.dashboard import Dashboard
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.glossary import Glossary
from metadata.generated.schema.entity.data.location import Location
from metadata.generated.schema.entity.data.metrics import Metrics
from metadata.generated.schema.entity.data.mlmodel import MlModel
@ -169,6 +170,11 @@ class OpenMetadata(
):
return "/mlmodels"
if issubclass(
entity, get_args(Union[Glossary, self.get_create_entity_type(Glossary)])
):
return "/glossary"
if issubclass(
entity, get_args(Union[Chart, self.get_create_entity_type(Chart)])
):

71
ingestion/src/metadata/ingestion/sink/elasticsearch.py
View File

@ -24,6 +24,7 @@ from metadata.config.common import ConfigModel
from metadata.generated.schema.entity.data.chart import Chart
from metadata.generated.schema.entity.data.dashboard import Dashboard
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.glossary import Glossary
from metadata.generated.schema.entity.data.pipeline import Pipeline, Task
from metadata.generated.schema.entity.data.table import Column, Table
from metadata.generated.schema.entity.data.topic import Topic
@ -39,6 +40,7 @@ from metadata.ingestion.api.sink import Sink, SinkStatus
from metadata.ingestion.models.table_metadata import (
ChangeDescription,
DashboardESDocument,
GlossaryESDocument,
PipelineESDocument,
TableESDocument,
TeamESDocument,
@ -49,6 +51,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
from metadata.ingestion.sink.elasticsearch_constants import (
DASHBOARD_ELASTICSEARCH_INDEX_MAPPING,
GLOSSARY_ELASTICSEARCH_INDEX_MAPPING,
PIPELINE_ELASTICSEARCH_INDEX_MAPPING,
TABLE_ELASTICSEARCH_INDEX_MAPPING,
TEAM_ELASTICSEARCH_INDEX_MAPPING,
@ -70,11 +73,13 @@ class ElasticSearchConfig(ConfigModel):
es_password: Optional[str] = None
index_tables: Optional[bool] = True
index_topics: Optional[bool] = True
index_glossary: Optional[bool] = True
index_dashboards: Optional[bool] = True
index_pipelines: Optional[bool] = True
index_users: Optional[bool] = True
index_teams: Optional[bool] = True
table_index_name: str = "table_search_index"
glossary_index_name: str = "glossary_search_index"
topic_index_name: str = "topic_search_index"
dashboard_index_name: str = "dashboard_search_index"
pipeline_index_name: str = "pipeline_search_index"
@ -139,6 +144,10 @@ class ElasticsearchSink(Sink[Entity]):
self._check_or_create_index(
self.config.table_index_name, TABLE_ELASTICSEARCH_INDEX_MAPPING
)
if self.config.index_glossary:
self._check_or_create_index(
self.config.glossary_index_name, GLOSSARY_ELASTICSEARCH_INDEX_MAPPING
)
if self.config.index_topics:
self._check_or_create_index(
self.config.topic_index_name, TOPIC_ELASTICSEARCH_INDEX_MAPPING
@ -201,6 +210,13 @@ class ElasticsearchSink(Sink[Entity]):
body=table_doc.json(),
request_timeout=self.config.timeout,
)
if isinstance(record, Glossary):
glossary_doc = self._create_glossary_es_doc(record)
self.elasticsearch_client.index(
index=self.config.glossary_index_name,
id=str(glossary_doc.glossary_id),
body=glossary_doc.json(),
)
if isinstance(record, Topic):
topic_doc = self._create_topic_es_doc(record)
self.elasticsearch_client.index(
@ -538,6 +554,61 @@ class ElasticsearchSink(Sink[Entity]):
return team_doc
def _create_glossary_es_doc(self, glossary: Glossary):
fqdn = glossary.fullyQualifiedName
suggest = [
{
"input": [
glossary.displayName if glossary.displayName else glossary.name
],
"weight": 10,
}
]
tags = set()
timestamp = time.time()
glossary_owner = (
str(glossary.owner.id.__root__) if glossary.owner is not None else ""
)
glossary_followers = []
if glossary.followers:
for follower in glossary.followers.__root__:
glossary_followers.append(str(follower.id.__root__))
tier = None
for glossary_tag in glossary.tags:
if "Tier" in glossary_tag.tagFQN:
tier = glossary_tag.tagFQN
else:
tags.add(glossary_tag.tagFQN)
# tasks: List[Task] = glossary.tasks # TODO Handle Glossary words
# task_names = []
# task_descriptions = []
# for task in tasks:
# task_names.append(task.displayName)
# if task.description is not None:
# task_descriptions.append(task.description)
# if tags in task and len(task.tags) > 0:
# for col_tag in task.tags:
# tags.add(col_tag.tagFQN)
glossary_doc = GlossaryESDocument(
glossary_id=str(glossary.id.__root__),
glossary_name=glossary.displayName
if glossary.displayName
else glossary.name,
# task_names=task_names, # TODO Handle Glossary words
# task_descriptions=task_descriptions,
suggest=suggest,
description=glossary.description,
last_updated_timestamp=timestamp,
tier=tier,
tags=list(tags),
fqdn=fqdn,
owner=glossary_owner,
followers=glossary_followers,
)
return glossary_doc
def _get_charts(self, chart_refs: Optional[List[entityReference.EntityReference]]):
charts = []
if chart_refs:

42
ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py
View File

@ -200,6 +200,48 @@ TOPIC_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
"""
)
GLOSSARY_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
"""
{
"mappings":{
"properties": {
"glossary_name": {
"type":"text"
},
"display_name": {
"type": "text"
},
"owner": {
"type": "text"
},
"followers": {
"type": "keyword"
},
"last_updated_timestamp": {
"type": "date",
"format": "epoch_second"
},
"description": {
"type": "text"
},
"tier": {
"type": "keyword"
},
"tags": {
"type": "keyword"
},
"entity_type": {
"type": "keyword"
},
"suggest": {
"type": "completion"
}
}
}
}
"""
)
DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
"""
{

25
ingestion/src/metadata/ingestion/source/metadata.py
View File

@ -24,7 +24,7 @@ from metadata.generated.schema.entity.teams.user import User
from metadata.ingestion.api.common import Entity, WorkflowContext
from metadata.ingestion.api.source import Source, SourceStatus
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
from metadata.ingestion.ometa.openmetadata_rest import Glossary, MetadataServerConfig
logger = logging.getLogger(__name__)
@ -35,6 +35,7 @@ class MetadataTablesRestSourceConfig(ConfigModel):
include_tables: Optional[bool] = True
include_topics: Optional[bool] = True
include_dashboards: Optional[bool] = True
include_glossary: Optional[bool] = True
include_pipelines: Optional[bool] = True
include_users: Optional[bool] = True
include_teams: Optional[bool] = True
@ -64,6 +65,10 @@ class MetadataSourceStatus(SourceStatus):
self.success.append(table_name)
logger.info("Table Scanned: %s", table_name)
def scanned_glossary(self, glossary_name: str) -> None:
self.success.append(glossary_name)
logger.info("Glossary Scanned: {}".format(glossary_name))
def scanned_topic(self, topic_name: str) -> None:
"""scanned topic method
@ -165,6 +170,7 @@ class MetadataSource(Source[Entity]):
def next_record(self) -> Iterable[Entity]:
yield from self.fetch_table()
yield from self.fetch_topic()
yield from self.fetch_glossary()
yield from self.fetch_dashboard()
yield from self.fetch_pipeline()
yield from self.fetch_users()
@ -315,6 +321,23 @@ class MetadataSource(Source[Entity]):
break
after = team_entities.after
def fetch_glossary(self) -> Glossary:
if self.config.include_glossary:
after = None
while True:
glossary_entities = self.metadata.list_entities(
entity=Glossary,
fields=["owner", "tags", "followers"],
after=after,
limit=self.config.limit_records,
)
for glossary in glossary_entities.entities:
self.status.scanned_glossary(glossary.name)
yield glossary
if glossary_entities.after is None:
break
after = glossary_entities.after
def get_status(self) -> SourceStatus:
return self.status

5
ingestion/tests/unit/test_ometa_endpoints.py
View File

@ -27,6 +27,7 @@ from metadata.generated.schema.entity.data.mlmodel import MlModel
from metadata.generated.schema.entity.data.pipeline import Pipeline
from metadata.generated.schema.entity.data.report import Report
from metadata.generated.schema.entity.data.table import Table
from metadata.generated.schema.entity.data.glossary import Glossary
from metadata.generated.schema.entity.data.topic import Topic
from metadata.generated.schema.entity.services.dashboardService import DashboardService
from metadata.generated.schema.entity.services.databaseService import DatabaseService
@ -51,6 +52,10 @@ class OMetaEndpointTest(TestCase):
Pass Entities and test their suffix generation
"""
# Glossary
self.assertEqual(self.metadata.get_suffix(Glossary), "/glossary")
# ML
self.assertEqual(self.metadata.get_suffix(MlModel), "/mlmodels")