mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-06-27 04:22:05 +00:00
Squash merge glossary support
This commit is contained in:
parent
52ec2e3ad1
commit
17adf5debc
13
bootstrap/sql/mysql/v004__create_db_connection_info.sql
Normal file
13
bootstrap/sql/mysql/v004__create_db_connection_info.sql
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
CREATE TABLE IF NOT EXISTS glossary_entity (
|
||||||
|
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||||
|
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||||
|
json JSON NOT NULL,
|
||||||
|
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||||
|
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||||
|
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
|
||||||
|
timestamp BIGINT,
|
||||||
|
PRIMARY KEY (id),
|
||||||
|
UNIQUE KEY unique_name(fullyQualifiedName),
|
||||||
|
INDEX (updatedBy),
|
||||||
|
INDEX (updatedAt)
|
||||||
|
);
|
@ -70,6 +70,7 @@ public final class Entity {
|
|||||||
public static final String UNUSED = "unused";
|
public static final String UNUSED = "unused";
|
||||||
public static final String BOTS = "bots";
|
public static final String BOTS = "bots";
|
||||||
public static final String LOCATION = "location";
|
public static final String LOCATION = "location";
|
||||||
|
public static final String GLOSSARY = "glossary";
|
||||||
|
|
||||||
//
|
//
|
||||||
// Policies
|
// Policies
|
||||||
|
@ -12,6 +12,7 @@ import java.util.Map;
|
|||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.function.Predicate;
|
import java.util.function.Predicate;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import javax.ws.rs.core.Response;
|
||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.EqualsAndHashCode;
|
import lombok.EqualsAndHashCode;
|
||||||
@ -31,6 +32,7 @@ import org.elasticsearch.client.indices.PutMappingRequest;
|
|||||||
import org.elasticsearch.common.xcontent.XContentType;
|
import org.elasticsearch.common.xcontent.XContentType;
|
||||||
import org.openmetadata.catalog.Entity;
|
import org.openmetadata.catalog.Entity;
|
||||||
import org.openmetadata.catalog.entity.data.Dashboard;
|
import org.openmetadata.catalog.entity.data.Dashboard;
|
||||||
|
import org.openmetadata.catalog.entity.data.Glossary;
|
||||||
import org.openmetadata.catalog.entity.data.Pipeline;
|
import org.openmetadata.catalog.entity.data.Pipeline;
|
||||||
import org.openmetadata.catalog.entity.data.Table;
|
import org.openmetadata.catalog.entity.data.Table;
|
||||||
import org.openmetadata.catalog.entity.data.Topic;
|
import org.openmetadata.catalog.entity.data.Topic;
|
||||||
@ -63,7 +65,8 @@ public class ElasticSearchIndexDefinition {
|
|||||||
TABLE_SEARCH_INDEX("table_search_index", "/elasticsearch/table_index_mapping.json"),
|
TABLE_SEARCH_INDEX("table_search_index", "/elasticsearch/table_index_mapping.json"),
|
||||||
TOPIC_SEARCH_INDEX("topic_search_index", "/elasticsearch/topic_index_mapping.json"),
|
TOPIC_SEARCH_INDEX("topic_search_index", "/elasticsearch/topic_index_mapping.json"),
|
||||||
DASHBOARD_SEARCH_INDEX("dashboard_search_index", "/elasticsearch/dashboard_index_mapping.json"),
|
DASHBOARD_SEARCH_INDEX("dashboard_search_index", "/elasticsearch/dashboard_index_mapping.json"),
|
||||||
PIPELINE_SEARCH_INDEX("pipeline_search_index", "/elasticsearch/pipeline_index_mapping.json");
|
PIPELINE_SEARCH_INDEX("pipeline_search_index", "/elasticsearch/pipeline_index_mapping.json"),
|
||||||
|
GLOSSARY_SEARCH_INDEX("glossary_search_index", "/elasticsearch/glossary_index_mapping.json");
|
||||||
|
|
||||||
public final String indexName;
|
public final String indexName;
|
||||||
public final String indexMappingFile;
|
public final String indexMappingFile;
|
||||||
@ -189,6 +192,8 @@ public class ElasticSearchIndexDefinition {
|
|||||||
return ElasticSearchIndexType.PIPELINE_SEARCH_INDEX;
|
return ElasticSearchIndexType.PIPELINE_SEARCH_INDEX;
|
||||||
} else if (type.equalsIgnoreCase(Entity.TOPIC)) {
|
} else if (type.equalsIgnoreCase(Entity.TOPIC)) {
|
||||||
return ElasticSearchIndexType.TOPIC_SEARCH_INDEX;
|
return ElasticSearchIndexType.TOPIC_SEARCH_INDEX;
|
||||||
|
} else if (type.equalsIgnoreCase(Entity.TOPIC)) {
|
||||||
|
return ElasticSearchIndexType.GLOSSARY_SEARCH_INDEX;
|
||||||
}
|
}
|
||||||
throw new RuntimeException("Failed to find index doc for type " + type);
|
throw new RuntimeException("Failed to find index doc for type " + type);
|
||||||
}
|
}
|
||||||
@ -707,3 +712,68 @@ class PipelineESIndex extends ElasticSearchIndex {
|
|||||||
return pipelineESIndexBuilder;
|
return pipelineESIndexBuilder;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@EqualsAndHashCode(callSuper = true)
|
||||||
|
@Getter
|
||||||
|
@SuperBuilder(builderMethodName = "internalBuilder")
|
||||||
|
@Value
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
|
class GlossaryESIndex extends ElasticSearchIndex {
|
||||||
|
@JsonProperty("glossary_id")
|
||||||
|
String glossaryId;
|
||||||
|
|
||||||
|
public static GlossaryESIndexBuilder builder(Glossary glossary, int responseCode) {
|
||||||
|
List<String> tags = new ArrayList<>();
|
||||||
|
List<String> taskNames = new ArrayList<>();
|
||||||
|
List<String> taskDescriptions = new ArrayList<>();
|
||||||
|
List<ElasticSearchSuggest> suggest = new ArrayList<>();
|
||||||
|
suggest.add(ElasticSearchSuggest.builder().input(glossary.getFullyQualifiedName()).weight(5).build());
|
||||||
|
suggest.add(ElasticSearchSuggest.builder().input(glossary.getDisplayName()).weight(10).build());
|
||||||
|
|
||||||
|
if (glossary.getTags() != null) {
|
||||||
|
glossary.getTags().forEach(tag -> tags.add(tag.getTagFQN()));
|
||||||
|
}
|
||||||
|
|
||||||
|
Long updatedTimestamp = glossary.getUpdatedAt();
|
||||||
|
ParseTags parseTags = new ParseTags(tags);
|
||||||
|
String description = glossary.getDescription() != null ? glossary.getDescription() : "";
|
||||||
|
String displayName = glossary.getDisplayName() != null ? glossary.getDisplayName() : "";
|
||||||
|
GlossaryESIndexBuilder builder =
|
||||||
|
internalBuilder()
|
||||||
|
.glossaryId(glossary.getId().toString())
|
||||||
|
.name(glossary.getDisplayName())
|
||||||
|
.displayName(description)
|
||||||
|
.description(displayName)
|
||||||
|
.fqdn(glossary.getFullyQualifiedName())
|
||||||
|
.lastUpdatedTimestamp(updatedTimestamp)
|
||||||
|
.entityType("glossary")
|
||||||
|
.suggest(suggest)
|
||||||
|
.tags(parseTags.tags)
|
||||||
|
.tier(parseTags.tierTag);
|
||||||
|
|
||||||
|
if (glossary.getFollowers() != null) {
|
||||||
|
builder.followers(
|
||||||
|
glossary.getFollowers().stream().map(item -> item.getId().toString()).collect(Collectors.toList()));
|
||||||
|
} else if (responseCode == Response.Status.CREATED.getStatusCode()) {
|
||||||
|
builder.followers(Collections.emptyList());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (glossary.getOwner() != null) {
|
||||||
|
builder.owner(glossary.getOwner().getId().toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
ESChangeDescription esChangeDescription = null;
|
||||||
|
if (glossary.getChangeDescription() != null) {
|
||||||
|
esChangeDescription =
|
||||||
|
ESChangeDescription.builder().updatedAt(updatedTimestamp).updatedBy(glossary.getUpdatedBy()).build();
|
||||||
|
esChangeDescription.setFieldsAdded(glossary.getChangeDescription().getFieldsAdded());
|
||||||
|
esChangeDescription.setFieldsDeleted(glossary.getChangeDescription().getFieldsDeleted());
|
||||||
|
esChangeDescription.setFieldsUpdated(glossary.getChangeDescription().getFieldsUpdated());
|
||||||
|
} else if (responseCode == Response.Status.CREATED.getStatusCode()) {
|
||||||
|
esChangeDescription =
|
||||||
|
ESChangeDescription.builder().updatedAt(updatedTimestamp).updatedBy(glossary.getUpdatedBy()).build();
|
||||||
|
}
|
||||||
|
builder.changeDescriptions(esChangeDescription != null ? List.of(esChangeDescription) : null);
|
||||||
|
return builder;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -33,6 +33,7 @@ import org.openmetadata.catalog.entity.Bots;
|
|||||||
import org.openmetadata.catalog.entity.data.Chart;
|
import org.openmetadata.catalog.entity.data.Chart;
|
||||||
import org.openmetadata.catalog.entity.data.Dashboard;
|
import org.openmetadata.catalog.entity.data.Dashboard;
|
||||||
import org.openmetadata.catalog.entity.data.Database;
|
import org.openmetadata.catalog.entity.data.Database;
|
||||||
|
import org.openmetadata.catalog.entity.data.Glossary;
|
||||||
import org.openmetadata.catalog.entity.data.Location;
|
import org.openmetadata.catalog.entity.data.Location;
|
||||||
import org.openmetadata.catalog.entity.data.Metrics;
|
import org.openmetadata.catalog.entity.data.Metrics;
|
||||||
import org.openmetadata.catalog.entity.data.MlModel;
|
import org.openmetadata.catalog.entity.data.MlModel;
|
||||||
@ -58,6 +59,7 @@ import org.openmetadata.catalog.jdbi3.DashboardRepository.DashboardEntityInterfa
|
|||||||
import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface;
|
import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface;
|
||||||
import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface;
|
import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface;
|
||||||
import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
|
import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
|
||||||
|
import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface;
|
||||||
import org.openmetadata.catalog.jdbi3.LocationRepository.LocationEntityInterface;
|
import org.openmetadata.catalog.jdbi3.LocationRepository.LocationEntityInterface;
|
||||||
import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface;
|
import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface;
|
||||||
import org.openmetadata.catalog.jdbi3.MetricsRepository.MetricsEntityInterface;
|
import org.openmetadata.catalog.jdbi3.MetricsRepository.MetricsEntityInterface;
|
||||||
@ -134,6 +136,9 @@ public interface CollectionDAO {
|
|||||||
@CreateSqlObject
|
@CreateSqlObject
|
||||||
MlModelDAO mlModelDAO();
|
MlModelDAO mlModelDAO();
|
||||||
|
|
||||||
|
@CreateSqlObject
|
||||||
|
GlossaryDAO glossaryDAO();
|
||||||
|
|
||||||
@CreateSqlObject
|
@CreateSqlObject
|
||||||
BotsDAO botsDAO();
|
BotsDAO botsDAO();
|
||||||
|
|
||||||
@ -689,6 +694,28 @@ public interface CollectionDAO {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface GlossaryDAO extends EntityDAO<Glossary> {
|
||||||
|
@Override
|
||||||
|
default String getTableName() {
|
||||||
|
return "glossary_entity";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
default Class<Glossary> getEntityClass() {
|
||||||
|
return Glossary.class;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
default String getNameColumn() {
|
||||||
|
return "fullyQualifiedName";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
default EntityReference getEntityReference(Glossary entity) {
|
||||||
|
return new GlossaryEntityInterface(entity).getEntityReference();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
interface AirflowPipelineDAO extends EntityDAO<AirflowPipeline> {
|
interface AirflowPipelineDAO extends EntityDAO<AirflowPipeline> {
|
||||||
@Override
|
@Override
|
||||||
default String getTableName() {
|
default String getTableName() {
|
||||||
|
@ -0,0 +1,271 @@
|
|||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.openmetadata.catalog.jdbi3;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.UUID;
|
||||||
|
import org.jdbi.v3.sqlobject.transaction.Transaction;
|
||||||
|
import org.openmetadata.catalog.Entity;
|
||||||
|
import org.openmetadata.catalog.entity.data.Glossary;
|
||||||
|
import org.openmetadata.catalog.resources.glossary.GlossaryResource;
|
||||||
|
import org.openmetadata.catalog.type.ChangeDescription;
|
||||||
|
import org.openmetadata.catalog.type.EntityReference;
|
||||||
|
import org.openmetadata.catalog.type.TagLabel;
|
||||||
|
import org.openmetadata.catalog.util.EntityInterface;
|
||||||
|
import org.openmetadata.catalog.util.EntityUtil;
|
||||||
|
import org.openmetadata.catalog.util.EntityUtil.Fields;
|
||||||
|
import org.openmetadata.catalog.util.JsonUtils;
|
||||||
|
|
||||||
|
public class GlossaryRepository extends EntityRepository<Glossary> {
|
||||||
|
private static final Fields GLOSSARY_UPDATE_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "owner,tags");
|
||||||
|
private static final Fields GLOSSARY_PATCH_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "owner,tags");
|
||||||
|
private final CollectionDAO dao;
|
||||||
|
|
||||||
|
public GlossaryRepository(CollectionDAO dao) {
|
||||||
|
super(
|
||||||
|
GlossaryResource.COLLECTION_PATH,
|
||||||
|
Entity.GLOSSARY,
|
||||||
|
Glossary.class,
|
||||||
|
dao.glossaryDAO(),
|
||||||
|
dao,
|
||||||
|
GLOSSARY_PATCH_FIELDS,
|
||||||
|
GLOSSARY_UPDATE_FIELDS,
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
false);
|
||||||
|
this.dao = dao;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getFQN(Glossary glossary) {
|
||||||
|
return (glossary.getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transaction
|
||||||
|
public EntityReference getOwnerReference(Glossary glossary) throws IOException {
|
||||||
|
return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), glossary.getOwner());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Glossary setFields(Glossary glossary, Fields fields) throws IOException, ParseException {
|
||||||
|
glossary.setDisplayName(glossary.getDisplayName());
|
||||||
|
glossary.setOwner(fields.contains("owner") ? getOwner(glossary) : null);
|
||||||
|
glossary.setFollowers(fields.contains("followers") ? getFollowers(glossary) : null);
|
||||||
|
glossary.setTags(fields.contains("tags") ? getTags(glossary.getFullyQualifiedName()) : null);
|
||||||
|
glossary.setUsageSummary(
|
||||||
|
fields.contains("usageSummary") ? EntityUtil.getLatestUsage(dao.usageDAO(), glossary.getId()) : null);
|
||||||
|
return glossary;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void prepare(Glossary glossary) throws IOException {
|
||||||
|
glossary.setFullyQualifiedName(getFQN(glossary));
|
||||||
|
EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), glossary.getOwner()); // Validate owner
|
||||||
|
glossary.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), glossary.getTags()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void storeEntity(Glossary glossary, boolean update) throws IOException {
|
||||||
|
// Relationships and fields such as href are derived and not stored as part of json
|
||||||
|
EntityReference owner = glossary.getOwner();
|
||||||
|
List<TagLabel> tags = glossary.getTags();
|
||||||
|
|
||||||
|
// Don't store owner, dashboard, href and tags as JSON. Build it on the fly based on relationships
|
||||||
|
glossary.withOwner(null).withHref(null).withTags(null);
|
||||||
|
|
||||||
|
if (update) {
|
||||||
|
dao.glossaryDAO().update(glossary.getId(), JsonUtils.pojoToJson(glossary));
|
||||||
|
} else {
|
||||||
|
dao.glossaryDAO().insert(glossary);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Restore the relationships
|
||||||
|
glossary.withOwner(owner).withTags(tags);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void restorePatchAttributes(Glossary original, Glossary updated) {}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EntityInterface<Glossary> getEntityInterface(Glossary entity) {
|
||||||
|
return new GlossaryEntityInterface(entity);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void storeRelationships(Glossary glossary) {
|
||||||
|
setOwner(glossary, glossary.getOwner());
|
||||||
|
applyTags(glossary);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EntityUpdater getUpdater(Glossary original, Glossary updated, Operation operation) {
|
||||||
|
return new GlossaryUpdater(original, updated, operation);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class GlossaryEntityInterface implements EntityInterface<Glossary> {
|
||||||
|
private final Glossary entity;
|
||||||
|
|
||||||
|
public GlossaryEntityInterface(Glossary entity) {
|
||||||
|
this.entity = entity;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public UUID getId() {
|
||||||
|
return entity.getId();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getDescription() {
|
||||||
|
return entity.getDescription();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getDisplayName() {
|
||||||
|
return entity.getDisplayName();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Boolean isDeleted() {
|
||||||
|
return entity.getDeleted();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EntityReference getOwner() {
|
||||||
|
return entity.getOwner();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getFullyQualifiedName() {
|
||||||
|
return entity.getFullyQualifiedName();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<TagLabel> getTags() {
|
||||||
|
return entity.getTags();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSkos() {
|
||||||
|
return entity.getSkos();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Double getVersion() {
|
||||||
|
return entity.getVersion();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getUpdatedBy() {
|
||||||
|
return entity.getUpdatedBy();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getUpdatedAt() {
|
||||||
|
return entity.getUpdatedAt();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public URI getHref() {
|
||||||
|
return entity.getHref();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<EntityReference> getFollowers() {
|
||||||
|
return entity.getFollowers();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ChangeDescription getChangeDescription() {
|
||||||
|
return entity.getChangeDescription();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EntityReference getEntityReference() {
|
||||||
|
return new EntityReference()
|
||||||
|
.withId(getId())
|
||||||
|
.withName(getFullyQualifiedName())
|
||||||
|
.withDescription(getDescription())
|
||||||
|
.withDisplayName(getDisplayName())
|
||||||
|
.withType(Entity.GLOSSARY);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Glossary getEntity() {
|
||||||
|
return entity;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EntityReference getContainer() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setId(UUID id) {
|
||||||
|
entity.setId(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setDescription(String description) {
|
||||||
|
entity.setDescription(description);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setDisplayName(String displayName) {
|
||||||
|
entity.setDisplayName(displayName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setUpdateDetails(String updatedBy, long updatedAt) {
|
||||||
|
entity.setUpdatedBy(updatedBy);
|
||||||
|
entity.setUpdatedAt(updatedAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setChangeDescription(Double newVersion, ChangeDescription changeDescription) {
|
||||||
|
entity.setVersion(newVersion);
|
||||||
|
entity.setChangeDescription(changeDescription);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setOwner(EntityReference owner) {
|
||||||
|
entity.setOwner(owner);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setDeleted(boolean flag) {
|
||||||
|
entity.setDeleted(flag);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Glossary withHref(URI href) {
|
||||||
|
return entity.withHref(href);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setTags(List<TagLabel> tags) {
|
||||||
|
entity.setTags(tags);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Handles entity updated from PUT and POST operation. */
|
||||||
|
public class GlossaryUpdater extends EntityUpdater {
|
||||||
|
public GlossaryUpdater(Glossary original, Glossary updated, Operation operation) {
|
||||||
|
super(original, updated, operation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,436 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2021 Collate
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.openmetadata.catalog.resources.glossary;
|
||||||
|
|
||||||
|
import com.google.inject.Inject;
|
||||||
|
import io.swagger.annotations.Api;
|
||||||
|
import io.swagger.v3.oas.annotations.ExternalDocumentation;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import io.swagger.v3.oas.annotations.Parameter;
|
||||||
|
import io.swagger.v3.oas.annotations.media.Content;
|
||||||
|
import io.swagger.v3.oas.annotations.media.ExampleObject;
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import io.swagger.v3.oas.annotations.parameters.RequestBody;
|
||||||
|
import io.swagger.v3.oas.annotations.responses.ApiResponse;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
|
import java.security.GeneralSecurityException;
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.UUID;
|
||||||
|
import javax.json.JsonPatch;
|
||||||
|
import javax.validation.Valid;
|
||||||
|
import javax.validation.constraints.Max;
|
||||||
|
import javax.validation.constraints.Min;
|
||||||
|
import javax.ws.rs.Consumes;
|
||||||
|
import javax.ws.rs.DELETE;
|
||||||
|
import javax.ws.rs.DefaultValue;
|
||||||
|
import javax.ws.rs.GET;
|
||||||
|
import javax.ws.rs.PATCH;
|
||||||
|
import javax.ws.rs.POST;
|
||||||
|
import javax.ws.rs.PUT;
|
||||||
|
import javax.ws.rs.Path;
|
||||||
|
import javax.ws.rs.PathParam;
|
||||||
|
import javax.ws.rs.Produces;
|
||||||
|
import javax.ws.rs.QueryParam;
|
||||||
|
import javax.ws.rs.core.Context;
|
||||||
|
import javax.ws.rs.core.MediaType;
|
||||||
|
import javax.ws.rs.core.Response;
|
||||||
|
import javax.ws.rs.core.SecurityContext;
|
||||||
|
import javax.ws.rs.core.UriInfo;
|
||||||
|
import org.openmetadata.catalog.Entity;
|
||||||
|
import org.openmetadata.catalog.api.data.CreateGlossary;
|
||||||
|
import org.openmetadata.catalog.entity.data.Glossary;
|
||||||
|
import org.openmetadata.catalog.jdbi3.CollectionDAO;
|
||||||
|
import org.openmetadata.catalog.jdbi3.GlossaryRepository;
|
||||||
|
import org.openmetadata.catalog.resources.Collection;
|
||||||
|
import org.openmetadata.catalog.security.Authorizer;
|
||||||
|
import org.openmetadata.catalog.security.SecurityUtil;
|
||||||
|
import org.openmetadata.catalog.type.EntityHistory;
|
||||||
|
import org.openmetadata.catalog.type.EntityReference;
|
||||||
|
import org.openmetadata.catalog.type.Include;
|
||||||
|
import org.openmetadata.catalog.util.EntityUtil.Fields;
|
||||||
|
import org.openmetadata.catalog.util.RestUtil;
|
||||||
|
import org.openmetadata.catalog.util.RestUtil.DeleteResponse;
|
||||||
|
import org.openmetadata.catalog.util.RestUtil.PatchResponse;
|
||||||
|
import org.openmetadata.catalog.util.RestUtil.PutResponse;
|
||||||
|
import org.openmetadata.catalog.util.ResultList;
|
||||||
|
|
||||||
|
@Path("/v1/glossary")
|
||||||
|
@Api(value = "Glossary collection", tags = "Glossary collection")
|
||||||
|
@Produces(MediaType.APPLICATION_JSON)
|
||||||
|
@Consumes(MediaType.APPLICATION_JSON)
|
||||||
|
@Collection(name = "glossary")
|
||||||
|
public class GlossaryResource {
|
||||||
|
public static final String COLLECTION_PATH = "v1/glossary/";
|
||||||
|
private final GlossaryRepository dao;
|
||||||
|
private final Authorizer authorizer;
|
||||||
|
|
||||||
|
public static void addHref(UriInfo uriInfo, EntityReference ref) {
|
||||||
|
ref.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, ref.getId()));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<Glossary> addHref(UriInfo uriInfo, List<Glossary> glossary) {
|
||||||
|
Optional.ofNullable(glossary).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
|
||||||
|
return glossary;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Glossary addHref(UriInfo uriInfo, Glossary glossary) {
|
||||||
|
glossary.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, glossary.getId()));
|
||||||
|
Entity.withHref(uriInfo, glossary.getOwner());
|
||||||
|
Entity.withHref(uriInfo, glossary.getFollowers());
|
||||||
|
return glossary;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
public GlossaryResource(CollectionDAO dao, Authorizer authorizer) {
|
||||||
|
Objects.requireNonNull(dao, "GlossaryRepository must not be null");
|
||||||
|
this.dao = new GlossaryRepository(dao);
|
||||||
|
this.authorizer = authorizer;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class GlossaryList extends ResultList<Glossary> {
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
GlossaryList() {
|
||||||
|
// Empty constructor needed for deserialization
|
||||||
|
}
|
||||||
|
|
||||||
|
public GlossaryList(List<Glossary> data, String beforeCursor, String afterCursor, int total)
|
||||||
|
throws GeneralSecurityException, UnsupportedEncodingException {
|
||||||
|
super(data, beforeCursor, afterCursor, total);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static final String FIELDS = "owner,dashboard,definition,followers,tags,usageSummary,skos";
|
||||||
|
public static final List<String> FIELD_LIST = Arrays.asList(FIELDS.replaceAll(" ", "").split(","));
|
||||||
|
|
||||||
|
@GET
|
||||||
|
@Valid
|
||||||
|
@Operation(
|
||||||
|
summary = "List Glossary",
|
||||||
|
tags = "glossary",
|
||||||
|
description =
|
||||||
|
"Get a list of glossary. Use `fields` parameter to get only necessary fields. "
|
||||||
|
+ " Use cursor-based pagination to limit the number "
|
||||||
|
+ "entries in the list using `limit` and `before` or `after` query params.",
|
||||||
|
responses = {
|
||||||
|
@ApiResponse(
|
||||||
|
responseCode = "200",
|
||||||
|
description = "List of glossary",
|
||||||
|
content = @Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryList.class)))
|
||||||
|
})
|
||||||
|
public ResultList<Glossary> list(
|
||||||
|
@Context UriInfo uriInfo,
|
||||||
|
@Context SecurityContext securityContext,
|
||||||
|
@Parameter(
|
||||||
|
description = "Fields requested in the returned resource",
|
||||||
|
schema = @Schema(type = "string", example = FIELDS))
|
||||||
|
@QueryParam("fields")
|
||||||
|
String fieldsParam,
|
||||||
|
@Parameter(description = "Limit the number glossary returned. (1 to 1000000, " + "default = 10)")
|
||||||
|
@DefaultValue("10")
|
||||||
|
@Min(1)
|
||||||
|
@Max(1000000)
|
||||||
|
@QueryParam("limit")
|
||||||
|
int limitParam,
|
||||||
|
@Parameter(description = "Returns list of glossary before this cursor", schema = @Schema(type = "string"))
|
||||||
|
@QueryParam("before")
|
||||||
|
String before,
|
||||||
|
@Parameter(description = "Returns list of glossary after this cursor", schema = @Schema(type = "string"))
|
||||||
|
@QueryParam("after")
|
||||||
|
String after,
|
||||||
|
@Parameter(
|
||||||
|
description = "Include all, deleted, or non-deleted entities.",
|
||||||
|
schema = @Schema(implementation = Include.class))
|
||||||
|
@QueryParam("include")
|
||||||
|
@DefaultValue("non-deleted")
|
||||||
|
Include include)
|
||||||
|
throws IOException, GeneralSecurityException, ParseException {
|
||||||
|
RestUtil.validateCursors(before, after);
|
||||||
|
Fields fields = new Fields(FIELD_LIST, fieldsParam);
|
||||||
|
|
||||||
|
ResultList<Glossary> glossary;
|
||||||
|
if (before != null) { // Reverse paging
|
||||||
|
glossary = dao.listBefore(uriInfo, fields, null, limitParam, before, include); // Ask for one extra entry
|
||||||
|
} else { // Forward paging or first page
|
||||||
|
glossary = dao.listAfter(uriInfo, fields, null, limitParam, after, include);
|
||||||
|
}
|
||||||
|
addHref(uriInfo, glossary.getData());
|
||||||
|
return glossary;
|
||||||
|
}
|
||||||
|
|
||||||
|
@GET
|
||||||
|
@Path("/{id}")
|
||||||
|
@Operation(
|
||||||
|
summary = "Get a glossary",
|
||||||
|
tags = "glossary",
|
||||||
|
description = "Get a glossary by `id`.",
|
||||||
|
responses = {
|
||||||
|
@ApiResponse(
|
||||||
|
responseCode = "200",
|
||||||
|
description = "The glossary",
|
||||||
|
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
|
||||||
|
@ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found")
|
||||||
|
})
|
||||||
|
public Glossary get(
|
||||||
|
@Context UriInfo uriInfo,
|
||||||
|
@Context SecurityContext securityContext,
|
||||||
|
@PathParam("id") String id,
|
||||||
|
@Parameter(
|
||||||
|
description = "Fields requested in the returned resource",
|
||||||
|
schema = @Schema(type = "string", example = FIELDS))
|
||||||
|
@QueryParam("fields")
|
||||||
|
String fieldsParam,
|
||||||
|
@Parameter(
|
||||||
|
description = "Include all, deleted, or non-deleted entities.",
|
||||||
|
schema = @Schema(implementation = Include.class))
|
||||||
|
@QueryParam("include")
|
||||||
|
@DefaultValue("non-deleted")
|
||||||
|
Include include)
|
||||||
|
throws IOException, ParseException {
|
||||||
|
Fields fields = new Fields(FIELD_LIST, fieldsParam);
|
||||||
|
return addHref(uriInfo, dao.get(uriInfo, id, fields, include));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GET
|
||||||
|
@Path("/name/{fqn}")
|
||||||
|
@Operation(
|
||||||
|
summary = "Get a glossary by name",
|
||||||
|
tags = "glossary",
|
||||||
|
description = "Get a glossary by fully qualified name.",
|
||||||
|
responses = {
|
||||||
|
@ApiResponse(
|
||||||
|
responseCode = "200",
|
||||||
|
description = "The glossary",
|
||||||
|
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
|
||||||
|
@ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found")
|
||||||
|
})
|
||||||
|
public Glossary getByName(
|
||||||
|
@Context UriInfo uriInfo,
|
||||||
|
@PathParam("fqn") String fqn,
|
||||||
|
@Context SecurityContext securityContext,
|
||||||
|
@Parameter(
|
||||||
|
description = "Fields requested in the returned resource",
|
||||||
|
schema = @Schema(type = "string", example = FIELDS))
|
||||||
|
@QueryParam("fields")
|
||||||
|
String fieldsParam,
|
||||||
|
@Parameter(
|
||||||
|
description = "Include all, deleted, or non-deleted entities.",
|
||||||
|
schema = @Schema(implementation = Include.class))
|
||||||
|
@QueryParam("include")
|
||||||
|
@DefaultValue("non-deleted")
|
||||||
|
Include include)
|
||||||
|
throws IOException, ParseException {
|
||||||
|
Fields fields = new Fields(FIELD_LIST, fieldsParam);
|
||||||
|
Glossary glossary = dao.getByName(uriInfo, fqn, fields, include);
|
||||||
|
return addHref(uriInfo, glossary);
|
||||||
|
}
|
||||||
|
|
||||||
|
@GET
|
||||||
|
@Path("/{id}/versions")
|
||||||
|
@Operation(
|
||||||
|
summary = "List glossary versions",
|
||||||
|
tags = "glossary",
|
||||||
|
description = "Get a list of all the versions of a glossary identified by `id`",
|
||||||
|
responses = {
|
||||||
|
@ApiResponse(
|
||||||
|
responseCode = "200",
|
||||||
|
description = "List of glossary versions",
|
||||||
|
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class)))
|
||||||
|
})
|
||||||
|
public EntityHistory listVersions(
|
||||||
|
@Context UriInfo uriInfo,
|
||||||
|
@Context SecurityContext securityContext,
|
||||||
|
@Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id)
|
||||||
|
throws IOException, ParseException {
|
||||||
|
return dao.listVersions(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
@GET
|
||||||
|
@Path("/{id}/versions/{version}")
|
||||||
|
@Operation(
|
||||||
|
summary = "Get a version of the glossary",
|
||||||
|
tags = "glossary",
|
||||||
|
description = "Get a version of the glossary by given `id`",
|
||||||
|
responses = {
|
||||||
|
@ApiResponse(
|
||||||
|
responseCode = "200",
|
||||||
|
description = "glossary",
|
||||||
|
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
|
||||||
|
@ApiResponse(
|
||||||
|
responseCode = "404",
|
||||||
|
description = "Glossary for instance {id} and version {version} is " + "not found")
|
||||||
|
})
|
||||||
|
public Glossary getVersion(
|
||||||
|
@Context UriInfo uriInfo,
|
||||||
|
@Context SecurityContext securityContext,
|
||||||
|
@Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id,
|
||||||
|
@Parameter(
|
||||||
|
description = "glossary version number in the form `major`.`minor`",
|
||||||
|
schema = @Schema(type = "string", example = "0.1 or 1.1"))
|
||||||
|
@PathParam("version")
|
||||||
|
String version)
|
||||||
|
throws IOException, ParseException {
|
||||||
|
return dao.getVersion(id, version);
|
||||||
|
}
|
||||||
|
|
||||||
|
@POST
|
||||||
|
@Operation(
|
||||||
|
summary = "Create a glossary",
|
||||||
|
tags = "glossary",
|
||||||
|
description = "Create a new glossary.",
|
||||||
|
responses = {
|
||||||
|
@ApiResponse(
|
||||||
|
responseCode = "200",
|
||||||
|
description = "The glossary",
|
||||||
|
content =
|
||||||
|
@Content(mediaType = "application/json", schema = @Schema(implementation = CreateGlossary.class))),
|
||||||
|
@ApiResponse(responseCode = "400", description = "Bad request")
|
||||||
|
})
|
||||||
|
public Response create(
|
||||||
|
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create)
|
||||||
|
throws IOException, ParseException {
|
||||||
|
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
|
||||||
|
Glossary glossary = getGlossary(securityContext, create);
|
||||||
|
glossary = addHref(uriInfo, dao.create(uriInfo, glossary));
|
||||||
|
return Response.created(glossary.getHref()).entity(glossary).build();
|
||||||
|
}
|
||||||
|
|
||||||
|
@PATCH
|
||||||
|
@Path("/{id}")
|
||||||
|
@Operation(
|
||||||
|
summary = "Update a glossary",
|
||||||
|
tags = "glossary",
|
||||||
|
description = "Update an existing glossary using JsonPatch.",
|
||||||
|
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
|
||||||
|
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
|
||||||
|
public Response updateDescription(
|
||||||
|
@Context UriInfo uriInfo,
|
||||||
|
@Context SecurityContext securityContext,
|
||||||
|
@PathParam("id") String id,
|
||||||
|
@RequestBody(
|
||||||
|
description = "JsonPatch with array of operations",
|
||||||
|
content =
|
||||||
|
@Content(
|
||||||
|
mediaType = MediaType.APPLICATION_JSON_PATCH_JSON,
|
||||||
|
examples = {
|
||||||
|
@ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]")
|
||||||
|
}))
|
||||||
|
JsonPatch patch)
|
||||||
|
throws IOException, ParseException {
|
||||||
|
Fields fields = new Fields(FIELD_LIST, FIELDS);
|
||||||
|
Glossary glossary = dao.get(uriInfo, id, fields);
|
||||||
|
SecurityUtil.checkAdminRoleOrPermissions(
|
||||||
|
authorizer, securityContext, dao.getEntityInterface(glossary).getEntityReference(), patch);
|
||||||
|
PatchResponse<Glossary> response =
|
||||||
|
dao.patch(uriInfo, UUID.fromString(id), securityContext.getUserPrincipal().getName(), patch);
|
||||||
|
addHref(uriInfo, response.getEntity());
|
||||||
|
return response.toResponse();
|
||||||
|
}
|
||||||
|
|
||||||
|
@PUT
|
||||||
|
@Operation(
|
||||||
|
summary = "Create or update a glossary",
|
||||||
|
tags = "glossary",
|
||||||
|
description = "Create a new glossary, if it does not exist or update an existing glossary.",
|
||||||
|
responses = {
|
||||||
|
@ApiResponse(
|
||||||
|
responseCode = "200",
|
||||||
|
description = "The glossary",
|
||||||
|
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
|
||||||
|
@ApiResponse(responseCode = "400", description = "Bad request")
|
||||||
|
})
|
||||||
|
public Response createOrUpdate(
|
||||||
|
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create)
|
||||||
|
throws IOException, ParseException {
|
||||||
|
Glossary glossary = getGlossary(securityContext, create);
|
||||||
|
PutResponse<Glossary> response = dao.createOrUpdate(uriInfo, glossary);
|
||||||
|
addHref(uriInfo, response.getEntity());
|
||||||
|
return response.toResponse();
|
||||||
|
}
|
||||||
|
|
||||||
|
@PUT
|
||||||
|
@Path("/{id}/followers")
|
||||||
|
@Operation(
|
||||||
|
summary = "Add a follower",
|
||||||
|
tags = "glossary",
|
||||||
|
description = "Add a user identified by `userId` as follower of this glossary",
|
||||||
|
responses = {
|
||||||
|
@ApiResponse(responseCode = "200", description = "OK"),
|
||||||
|
@ApiResponse(responseCode = "404", description = "glossary for instance {id} is not found")
|
||||||
|
})
|
||||||
|
public Response addFollower(
|
||||||
|
@Context UriInfo uriInfo,
|
||||||
|
@Context SecurityContext securityContext,
|
||||||
|
@Parameter(description = "Id of the glossary", schema = @Schema(type = "string")) @PathParam("id") String id,
|
||||||
|
@Parameter(description = "Id of the user to be added as follower", schema = @Schema(type = "string"))
|
||||||
|
String userId)
|
||||||
|
throws IOException, ParseException {
|
||||||
|
return dao.addFollower(securityContext.getUserPrincipal().getName(), UUID.fromString(id), UUID.fromString(userId))
|
||||||
|
.toResponse();
|
||||||
|
}
|
||||||
|
|
||||||
|
@DELETE
|
||||||
|
@Path("/{id}/followers/{userId}")
|
||||||
|
@Operation(
|
||||||
|
summary = "Remove a follower",
|
||||||
|
tags = "glossary",
|
||||||
|
description = "Remove the user identified `userId` as a follower of the glossary.")
|
||||||
|
public Response deleteFollower(
|
||||||
|
@Context UriInfo uriInfo,
|
||||||
|
@Context SecurityContext securityContext,
|
||||||
|
@Parameter(description = "Id of the glossary", schema = @Schema(type = "string")) @PathParam("id") String id,
|
||||||
|
@Parameter(description = "Id of the user being removed as follower", schema = @Schema(type = "string"))
|
||||||
|
@PathParam("userId")
|
||||||
|
String userId)
|
||||||
|
throws IOException, ParseException {
|
||||||
|
return dao.deleteFollower(
|
||||||
|
securityContext.getUserPrincipal().getName(), UUID.fromString(id), UUID.fromString(userId))
|
||||||
|
.toResponse();
|
||||||
|
}
|
||||||
|
|
||||||
|
@DELETE
|
||||||
|
@Path("/{id}")
|
||||||
|
@Operation(
|
||||||
|
summary = "Delete a Glossary",
|
||||||
|
tags = "glossary",
|
||||||
|
description = "Delete a glossary by `id`.",
|
||||||
|
responses = {
|
||||||
|
@ApiResponse(responseCode = "200", description = "OK"),
|
||||||
|
@ApiResponse(responseCode = "404", description = "glossary for instance {id} is not found")
|
||||||
|
})
|
||||||
|
public Response delete(@Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("id") String id)
|
||||||
|
throws IOException, ParseException {
|
||||||
|
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
|
||||||
|
DeleteResponse<Glossary> response = dao.delete(securityContext.getUserPrincipal().getName(), id);
|
||||||
|
return response.toResponse();
|
||||||
|
}
|
||||||
|
|
||||||
|
private Glossary getGlossary(SecurityContext securityContext, CreateGlossary create) {
|
||||||
|
return new Glossary()
|
||||||
|
.withId(UUID.randomUUID())
|
||||||
|
.withName(create.getName())
|
||||||
|
.withDisplayName(create.getDisplayName())
|
||||||
|
.withDescription(create.getDescription())
|
||||||
|
.withSkos(create.getSkos())
|
||||||
|
.withTags(create.getTags())
|
||||||
|
.withOwner(create.getOwner())
|
||||||
|
.withUpdatedBy(securityContext.getUserPrincipal().getName())
|
||||||
|
.withUpdatedAt(System.currentTimeMillis());
|
||||||
|
}
|
||||||
|
}
|
@ -146,6 +146,9 @@ public class SearchResource {
|
|||||||
case "table_search_index":
|
case "table_search_index":
|
||||||
searchSourceBuilder = buildTableSearchBuilder(query, from, size);
|
searchSourceBuilder = buildTableSearchBuilder(query, from, size);
|
||||||
break;
|
break;
|
||||||
|
case "glossary_search_index":
|
||||||
|
searchSourceBuilder = buildGlossarySearchBuilder(query, from, size);
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
searchSourceBuilder = buildAggregateSearchBuilder(query, from, size);
|
searchSourceBuilder = buildAggregateSearchBuilder(query, from, size);
|
||||||
break;
|
break;
|
||||||
@ -353,4 +356,27 @@ public class SearchResource {
|
|||||||
|
|
||||||
return searchSourceBuilder;
|
return searchSourceBuilder;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private SearchSourceBuilder buildGlossarySearchBuilder(String query, int from, int size) {
|
||||||
|
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||||
|
HighlightBuilder.Field highlightGlossaryName = new HighlightBuilder.Field("glossary_name");
|
||||||
|
highlightGlossaryName.highlighterType("unified");
|
||||||
|
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field("description");
|
||||||
|
highlightDescription.highlighterType("unified");
|
||||||
|
HighlightBuilder hb = new HighlightBuilder();
|
||||||
|
hb.field(highlightDescription);
|
||||||
|
hb.field(highlightGlossaryName);
|
||||||
|
hb.preTags("<span class=\"text-highlighter\">");
|
||||||
|
hb.postTags("</span>");
|
||||||
|
searchSourceBuilder
|
||||||
|
.query(QueryBuilders.queryStringQuery(query).field("glossary_name", 5.0f).field("description").lenient(true))
|
||||||
|
.aggregation(AggregationBuilders.terms("EntityType").field("entity_type"))
|
||||||
|
.aggregation(AggregationBuilders.terms("Tier").field("tier"))
|
||||||
|
.aggregation(AggregationBuilders.terms("Tags").field("tags"))
|
||||||
|
.highlighter(hb)
|
||||||
|
.from(from)
|
||||||
|
.size(size);
|
||||||
|
|
||||||
|
return searchSourceBuilder;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,40 @@
|
|||||||
|
{
|
||||||
|
"$id": "https://open-metadata.org/schema/api/data/createGlossary.json",
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "Create Glossary entity request",
|
||||||
|
"description": "Create Glossary entity request",
|
||||||
|
"type": "object",
|
||||||
|
"properties" : {
|
||||||
|
"name": {
|
||||||
|
"description": "Name that identifies this glossary.",
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1,
|
||||||
|
"maxLength": 128
|
||||||
|
},
|
||||||
|
"displayName": {
|
||||||
|
"description": "Display Name that identifies this glossary.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"description": {
|
||||||
|
"description": "Description of the glossary instance.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"skos": {
|
||||||
|
"description": "SKOS data in JSON-LD format",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"tags": {
|
||||||
|
"description": "Tags for this glossary",
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "../../type/tagLabel.json"
|
||||||
|
},
|
||||||
|
"default": null
|
||||||
|
},
|
||||||
|
"owner": {
|
||||||
|
"description": "Owner of this glossary",
|
||||||
|
"$ref": "../../type/entityReference.json"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["name"]
|
||||||
|
}
|
@ -0,0 +1,85 @@
|
|||||||
|
{
|
||||||
|
"$id": "https://open-metadata.org/schema/entity/data/glossary.json",
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "Glossary",
|
||||||
|
"description": "This schema defines the Glossary entity based on SKOS.",
|
||||||
|
"type": "object",
|
||||||
|
|
||||||
|
"properties" : {
|
||||||
|
"id": {
|
||||||
|
"description": "Unique identifier of a glossary instance.",
|
||||||
|
"$ref": "../../type/basic.json#/definitions/uuid"
|
||||||
|
},
|
||||||
|
"name": {
|
||||||
|
"description": "Name that identifies this glossary.",
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1,
|
||||||
|
"maxLength": 128
|
||||||
|
},
|
||||||
|
"fullyQualifiedName": {
|
||||||
|
"description": "A unique name that identifies a glossary.",
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1,
|
||||||
|
"maxLength": 256
|
||||||
|
},
|
||||||
|
"displayName": {
|
||||||
|
"description": "Display Name that identifies this glossary.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"version" : {
|
||||||
|
"description": "Metadata version of the entity.",
|
||||||
|
"$ref": "../../type/entityHistory.json#/definitions/entityVersion"
|
||||||
|
},
|
||||||
|
"updatedAt" : {
|
||||||
|
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
|
||||||
|
"$ref": "../../type/basic.json#/definitions/timestamp"
|
||||||
|
},
|
||||||
|
"updatedBy" : {
|
||||||
|
"description": "User who made the update.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"description": {
|
||||||
|
"description": "Description of the glossary.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"skos": {
|
||||||
|
"description": "SKOS data in JSON-LD format",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"href": {
|
||||||
|
"description": "Link to the resource corresponding to this entity.",
|
||||||
|
"$ref": "../../type/basic.json#/definitions/href"
|
||||||
|
},
|
||||||
|
"owner": {
|
||||||
|
"description": "Owner of this glossary.",
|
||||||
|
"$ref": "../../type/entityReference.json"
|
||||||
|
},
|
||||||
|
"followers": {
|
||||||
|
"description": "Followers of this glossary.",
|
||||||
|
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList"
|
||||||
|
},
|
||||||
|
"tags": {
|
||||||
|
"description": "Tags for this glossary.",
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "../../type/tagLabel.json"
|
||||||
|
},
|
||||||
|
"default": null
|
||||||
|
},
|
||||||
|
"usageSummary" : {
|
||||||
|
"description": "Latest usage information for this glossary.",
|
||||||
|
"$ref": "../../type/usageDetails.json",
|
||||||
|
"default": null
|
||||||
|
},
|
||||||
|
"changeDescription": {
|
||||||
|
"description" : "Change that lead to this version of the entity.",
|
||||||
|
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
|
||||||
|
},
|
||||||
|
"deleted" : {
|
||||||
|
"description": "When `true` indicates the entity has been soft deleted.",
|
||||||
|
"type" : "boolean",
|
||||||
|
"default": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["id", "name"]
|
||||||
|
}
|
@ -0,0 +1,259 @@
|
|||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.openmetadata.catalog.resources.glossary;
|
||||||
|
|
||||||
|
import static javax.ws.rs.core.Response.Status.FORBIDDEN;
|
||||||
|
import static javax.ws.rs.core.Response.Status.NOT_FOUND;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.openmetadata.catalog.security.SecurityUtil.authHeaders;
|
||||||
|
import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
|
||||||
|
import static org.openmetadata.catalog.util.TestUtils.NON_EXISTENT_ENTITY;
|
||||||
|
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
import org.apache.http.client.HttpResponseException;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.MethodOrderer;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.TestInfo;
|
||||||
|
import org.junit.jupiter.api.TestMethodOrder;
|
||||||
|
import org.openmetadata.catalog.CatalogApplicationTest;
|
||||||
|
import org.openmetadata.catalog.Entity;
|
||||||
|
import org.openmetadata.catalog.api.data.CreateGlossary;
|
||||||
|
import org.openmetadata.catalog.entity.data.Glossary;
|
||||||
|
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
|
||||||
|
import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface;
|
||||||
|
import org.openmetadata.catalog.resources.EntityResourceTest;
|
||||||
|
import org.openmetadata.catalog.type.EntityReference;
|
||||||
|
import org.openmetadata.catalog.util.TestUtils;
|
||||||
|
|
||||||
|
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||||
|
public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlossary> {
|
||||||
|
public GlossaryResourceTest() {
|
||||||
|
super(
|
||||||
|
Entity.GLOSSARY,
|
||||||
|
Glossary.class,
|
||||||
|
GlossaryResource.GlossaryList.class,
|
||||||
|
"glossary",
|
||||||
|
GlossaryResource.FIELDS,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
public void setup(TestInfo test) throws IOException, URISyntaxException {
|
||||||
|
super.setup(test);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public CreateGlossary createRequest(String name, String description, String displayName, EntityReference owner) {
|
||||||
|
return new CreateGlossary()
|
||||||
|
.withName(name)
|
||||||
|
.withDescription(description)
|
||||||
|
.withDisplayName(displayName)
|
||||||
|
.withOwner(owner);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EntityReference getContainer(CreateGlossary createRequest) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Entity tests
|
||||||
|
|
||||||
|
// @Test
|
||||||
|
// public void patch_entityAttributes_200_ok()
|
||||||
|
// {}
|
||||||
|
|
||||||
|
// @Test
|
||||||
|
// public void put_entityCreate_200()
|
||||||
|
// {}
|
||||||
|
|
||||||
|
// @Test
|
||||||
|
// public void put_entityCreate_as_owner_200()
|
||||||
|
// {}
|
||||||
|
|
||||||
|
// @Test
|
||||||
|
// public void put_entityEmptyDescriptionUpdate_200()
|
||||||
|
// {}
|
||||||
|
|
||||||
|
// @Test
|
||||||
|
// public void put_entityNullDescriptionUpdate_200()
|
||||||
|
// {}
|
||||||
|
|
||||||
|
// @Test
|
||||||
|
// public void put_entityUpdateOwner_200()
|
||||||
|
// {}
|
||||||
|
|
||||||
|
// @Test
|
||||||
|
// public void put_entityUpdateWithNoChange_200()
|
||||||
|
// {}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void post_validGlossary_200_OK(TestInfo test) throws IOException {
|
||||||
|
CreateGlossary create = create(test).withDescription("description");
|
||||||
|
createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void post_glossaryWithUserOwner_200_ok(TestInfo test) throws IOException {
|
||||||
|
createAndCheckEntity(create(test).withOwner(USER_OWNER1), ADMIN_AUTH_HEADERS);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void post_glossaryWithTeamOwner_200_ok(TestInfo test) throws IOException {
|
||||||
|
createAndCheckEntity(create(test).withOwner(TEAM_OWNER1), ADMIN_AUTH_HEADERS);
|
||||||
|
}
|
||||||
|
|
||||||
|
// @Test
|
||||||
|
// public void post_glossaryWithInvalidOwnerType_4xx(TestInfo test) {
|
||||||
|
// EntityReference owner = new EntityReference().withId(TEAM1.getId()); /* No owner type is set */
|
||||||
|
// CreateGlossary create = create(test).withOwner(owner);
|
||||||
|
// HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
|
||||||
|
// createEntity(create, ADMIN_AUTH_HEADERS));
|
||||||
|
// TestUtils.assertResponseContains(exception, BAD_REQUEST, "type must not be null");
|
||||||
|
// }
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void post_glossary_as_non_admin_401(TestInfo test) {
|
||||||
|
CreateGlossary create = create(test);
|
||||||
|
HttpResponseException exception =
|
||||||
|
assertThrows(HttpResponseException.class, () -> createEntity(create, authHeaders("test@open-metadata.org")));
|
||||||
|
assertResponse(exception, FORBIDDEN, "Principal: CatalogPrincipal{name='test'} is not admin");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void get_nonExistentGlossary_404_notFound() {
|
||||||
|
HttpResponseException exception =
|
||||||
|
assertThrows(HttpResponseException.class, () -> getEntity(NON_EXISTENT_ENTITY, ADMIN_AUTH_HEADERS));
|
||||||
|
assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, NON_EXISTENT_ENTITY));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void delete_glossary_200_ok(TestInfo test) throws HttpResponseException {
|
||||||
|
Glossary glossary = createEntity(create(test), ADMIN_AUTH_HEADERS);
|
||||||
|
deleteGlossary(glossary.getId(), ADMIN_AUTH_HEADERS);
|
||||||
|
}
|
||||||
|
|
||||||
|
// @Test
|
||||||
|
// public void delete_glossary_as_non_admin_401(TestInfo test) throws HttpResponseException {
|
||||||
|
// Glossary glossary = createEntity(create(test), ADMIN_AUTH_HEADERS);
|
||||||
|
// HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
|
||||||
|
// deleteGlossary(glossary.getId(), authHeaders("test@open-metadata.org")));
|
||||||
|
// assertResponse(exception, FORBIDDEN, "Principal: CatalogPrincipal{name='test'} is not admin");
|
||||||
|
// }
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void delete_nonExistentGlossary_404() {
|
||||||
|
HttpResponseException exception =
|
||||||
|
assertThrows(HttpResponseException.class, () -> getEntity(NON_EXISTENT_ENTITY, ADMIN_AUTH_HEADERS));
|
||||||
|
assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, NON_EXISTENT_ENTITY));
|
||||||
|
}
|
||||||
|
|
||||||
|
// @Test
|
||||||
|
// public void put_GlossaryCreate_200(TestInfo test) throws HttpResponseException {
|
||||||
|
// // Create a new Glossary with PUT
|
||||||
|
// CreateGlossary request = create(test).withOwner(USER_OWNER1);
|
||||||
|
// updateAndCheckGlossary(null, request.withName(test.getDisplayName()).withDescription(null), CREATED,
|
||||||
|
// ADMIN_AUTH_HEADERS, NO_CHANGE);
|
||||||
|
// }
|
||||||
|
|
||||||
|
public static CreateGlossary create(TestInfo test) {
|
||||||
|
return create(test, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static CreateGlossary create(TestInfo test, int index) {
|
||||||
|
return new CreateGlossary().withName(getGlossaryName(test, index));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A method variant to be called form other tests to create a glossary without depending on Database, DatabaseService
|
||||||
|
* set up in the {@code setup()} method
|
||||||
|
*/
|
||||||
|
public Glossary createEntity(TestInfo test, int index) throws IOException {
|
||||||
|
CreateGlossary create = new CreateGlossary().withName(getGlossaryName(test, index));
|
||||||
|
return createEntity(create, ADMIN_AUTH_HEADERS);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteGlossary(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
|
||||||
|
TestUtils.delete(CatalogApplicationTest.getResource("glossary/" + id), authHeaders);
|
||||||
|
|
||||||
|
// Check to make sure database entity does not exist
|
||||||
|
HttpResponseException exception = assertThrows(HttpResponseException.class, () -> getEntity(id, authHeaders));
|
||||||
|
assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, id));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getGlossaryName(TestInfo test, int index) {
|
||||||
|
return String.format("glossary%d_%s", index, test.getDisplayName());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void validateCreatedEntity(
|
||||||
|
Glossary createdEntity, CreateGlossary createRequest, Map<String, String> authHeaders)
|
||||||
|
throws HttpResponseException {
|
||||||
|
validateCommonEntityFields(
|
||||||
|
getEntityInterface(createdEntity),
|
||||||
|
createRequest.getDescription(),
|
||||||
|
TestUtils.getPrincipal(authHeaders),
|
||||||
|
createRequest.getOwner());
|
||||||
|
|
||||||
|
// Entity specific validation
|
||||||
|
TestUtils.validateTags(createRequest.getTags(), createdEntity.getTags());
|
||||||
|
TestUtils.validateEntityReference(createdEntity.getFollowers());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void validateUpdatedEntity(Glossary updated, CreateGlossary request, Map<String, String> authHeaders)
|
||||||
|
throws HttpResponseException {
|
||||||
|
validateCreatedEntity(updated, request, authHeaders);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void compareEntities(Glossary expected, Glossary patched, Map<String, String> authHeaders)
|
||||||
|
throws HttpResponseException {
|
||||||
|
validateCommonEntityFields(
|
||||||
|
getEntityInterface(patched),
|
||||||
|
expected.getDescription(),
|
||||||
|
TestUtils.getPrincipal(authHeaders),
|
||||||
|
expected.getOwner());
|
||||||
|
|
||||||
|
// Entity specific validation
|
||||||
|
TestUtils.validateTags(expected.getTags(), patched.getTags());
|
||||||
|
TestUtils.validateEntityReference(expected.getFollowers());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GlossaryEntityInterface getEntityInterface(Glossary entity) {
|
||||||
|
return new GlossaryEntityInterface(entity);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void validateGetWithDifferentFields(Glossary entity, boolean byName) throws HttpResponseException {}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
|
||||||
|
if (expected == actual) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
assertCommonFieldChange(fieldName, expected, actual);
|
||||||
|
}
|
||||||
|
}
|
1332
ingestion/examples/sample_data/thesauruses/hobbies-SKOS.json
Normal file
1332
ingestion/examples/sample_data/thesauruses/hobbies-SKOS.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -5,6 +5,7 @@
|
|||||||
"include_tables": "true",
|
"include_tables": "true",
|
||||||
"include_topics": "true",
|
"include_topics": "true",
|
||||||
"include_dashboards": "true",
|
"include_dashboards": "true",
|
||||||
|
"include_glossary": "true",
|
||||||
"limit_records": 10
|
"limit_records": 10
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@ -14,6 +15,7 @@
|
|||||||
"index_tables": "true",
|
"index_tables": "true",
|
||||||
"index_topics": "true",
|
"index_topics": "true",
|
||||||
"index_dashboards": "true",
|
"index_dashboards": "true",
|
||||||
|
"index_glossary": "true",
|
||||||
"es_host": "localhost",
|
"es_host": "localhost",
|
||||||
"es_port": 9200
|
"es_port": 9200
|
||||||
}
|
}
|
||||||
|
@ -104,6 +104,23 @@ class TopicESDocument(BaseModel):
|
|||||||
doc_as_upsert: bool = True
|
doc_as_upsert: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
class GlossaryESDocument(BaseModel):
|
||||||
|
"""Glossary Elastic Search Mapping doc"""
|
||||||
|
|
||||||
|
glossary_id: str
|
||||||
|
glossary_name: str
|
||||||
|
entity_type: str = "glossary"
|
||||||
|
suggest: List[dict]
|
||||||
|
description: Optional[str] = None
|
||||||
|
last_updated_timestamp: Optional[int]
|
||||||
|
tags: List[str]
|
||||||
|
fqdn: str
|
||||||
|
tier: Optional[str] = None
|
||||||
|
schema_description: Optional[str] = None
|
||||||
|
owner: str
|
||||||
|
followers: List[str]
|
||||||
|
|
||||||
|
|
||||||
class DashboardESDocument(BaseModel):
|
class DashboardESDocument(BaseModel):
|
||||||
"""Elastic Search Mapping doc for Dashboards"""
|
"""Elastic Search Mapping doc for Dashboards"""
|
||||||
|
|
||||||
|
@ -31,6 +31,7 @@ from metadata.generated.schema.entity.data.mlmodel import MlModel
|
|||||||
from metadata.generated.schema.entity.data.pipeline import Pipeline
|
from metadata.generated.schema.entity.data.pipeline import Pipeline
|
||||||
from metadata.generated.schema.entity.data.report import Report
|
from metadata.generated.schema.entity.data.report import Report
|
||||||
from metadata.generated.schema.entity.data.table import Table
|
from metadata.generated.schema.entity.data.table import Table
|
||||||
|
from metadata.generated.schema.entity.data.glossary import Glossary
|
||||||
from metadata.generated.schema.entity.data.topic import Topic
|
from metadata.generated.schema.entity.data.topic import Topic
|
||||||
from metadata.generated.schema.entity.policies.policy import Policy
|
from metadata.generated.schema.entity.policies.policy import Policy
|
||||||
from metadata.generated.schema.entity.services.dashboardService import DashboardService
|
from metadata.generated.schema.entity.services.dashboardService import DashboardService
|
||||||
@ -162,6 +163,11 @@ class OpenMetadata(
|
|||||||
):
|
):
|
||||||
return "/mlmodels"
|
return "/mlmodels"
|
||||||
|
|
||||||
|
if issubclass(
|
||||||
|
entity, get_args(Union[Glossary, self.get_create_entity_type(Glossary)])
|
||||||
|
):
|
||||||
|
return "/glossary"
|
||||||
|
|
||||||
if issubclass(
|
if issubclass(
|
||||||
entity, get_args(Union[Chart, self.get_create_entity_type(Chart)])
|
entity, get_args(Union[Chart, self.get_create_entity_type(Chart)])
|
||||||
):
|
):
|
||||||
|
@ -26,6 +26,7 @@ from metadata.generated.schema.entity.data.dashboard import Dashboard
|
|||||||
from metadata.generated.schema.entity.data.database import Database
|
from metadata.generated.schema.entity.data.database import Database
|
||||||
from metadata.generated.schema.entity.data.pipeline import Pipeline, Task
|
from metadata.generated.schema.entity.data.pipeline import Pipeline, Task
|
||||||
from metadata.generated.schema.entity.data.table import Column, Table
|
from metadata.generated.schema.entity.data.table import Column, Table
|
||||||
|
from metadata.generated.schema.entity.data.glossary import Glossary
|
||||||
from metadata.generated.schema.entity.data.topic import Topic
|
from metadata.generated.schema.entity.data.topic import Topic
|
||||||
from metadata.generated.schema.entity.services.dashboardService import DashboardService
|
from metadata.generated.schema.entity.services.dashboardService import DashboardService
|
||||||
from metadata.generated.schema.entity.services.databaseService import DatabaseService
|
from metadata.generated.schema.entity.services.databaseService import DatabaseService
|
||||||
@ -39,6 +40,7 @@ from metadata.ingestion.models.table_metadata import (
|
|||||||
DashboardESDocument,
|
DashboardESDocument,
|
||||||
PipelineESDocument,
|
PipelineESDocument,
|
||||||
TableESDocument,
|
TableESDocument,
|
||||||
|
GlossaryESDocument,
|
||||||
TopicESDocument,
|
TopicESDocument,
|
||||||
)
|
)
|
||||||
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
||||||
@ -47,6 +49,7 @@ from metadata.ingestion.sink.elasticsearch_constants import (
|
|||||||
DASHBOARD_ELASTICSEARCH_INDEX_MAPPING,
|
DASHBOARD_ELASTICSEARCH_INDEX_MAPPING,
|
||||||
PIPELINE_ELASTICSEARCH_INDEX_MAPPING,
|
PIPELINE_ELASTICSEARCH_INDEX_MAPPING,
|
||||||
TABLE_ELASTICSEARCH_INDEX_MAPPING,
|
TABLE_ELASTICSEARCH_INDEX_MAPPING,
|
||||||
|
GLOSSARY_ELASTICSEARCH_INDEX_MAPPING,
|
||||||
TOPIC_ELASTICSEARCH_INDEX_MAPPING,
|
TOPIC_ELASTICSEARCH_INDEX_MAPPING,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -64,10 +67,12 @@ class ElasticSearchConfig(ConfigModel):
|
|||||||
es_password: Optional[str] = None
|
es_password: Optional[str] = None
|
||||||
index_tables: Optional[bool] = True
|
index_tables: Optional[bool] = True
|
||||||
index_topics: Optional[bool] = True
|
index_topics: Optional[bool] = True
|
||||||
|
index_glossary: Optional[bool] = True
|
||||||
index_dashboards: Optional[bool] = True
|
index_dashboards: Optional[bool] = True
|
||||||
index_pipelines: Optional[bool] = True
|
index_pipelines: Optional[bool] = True
|
||||||
index_dbt_models: Optional[bool] = True
|
index_dbt_models: Optional[bool] = True
|
||||||
table_index_name: str = "table_search_index"
|
table_index_name: str = "table_search_index"
|
||||||
|
glossary_index_name: str = "glossary_search_index"
|
||||||
topic_index_name: str = "topic_search_index"
|
topic_index_name: str = "topic_search_index"
|
||||||
dashboard_index_name: str = "dashboard_search_index"
|
dashboard_index_name: str = "dashboard_search_index"
|
||||||
pipeline_index_name: str = "pipeline_search_index"
|
pipeline_index_name: str = "pipeline_search_index"
|
||||||
@ -131,6 +136,10 @@ class ElasticsearchSink(Sink[Entity]):
|
|||||||
self._check_or_create_index(
|
self._check_or_create_index(
|
||||||
self.config.table_index_name, TABLE_ELASTICSEARCH_INDEX_MAPPING
|
self.config.table_index_name, TABLE_ELASTICSEARCH_INDEX_MAPPING
|
||||||
)
|
)
|
||||||
|
if self.config.index_glossary:
|
||||||
|
self._check_or_create_index(
|
||||||
|
self.config.glossary_index_name, GLOSSARY_ELASTICSEARCH_INDEX_MAPPING
|
||||||
|
)
|
||||||
if self.config.index_topics:
|
if self.config.index_topics:
|
||||||
self._check_or_create_index(
|
self._check_or_create_index(
|
||||||
self.config.topic_index_name, TOPIC_ELASTICSEARCH_INDEX_MAPPING
|
self.config.topic_index_name, TOPIC_ELASTICSEARCH_INDEX_MAPPING
|
||||||
@ -183,6 +192,13 @@ class ElasticsearchSink(Sink[Entity]):
|
|||||||
body=table_doc.json(),
|
body=table_doc.json(),
|
||||||
request_timeout=self.config.timeout,
|
request_timeout=self.config.timeout,
|
||||||
)
|
)
|
||||||
|
if isinstance(record, Glossary):
|
||||||
|
glossary_doc = self._create_glossary_es_doc(record)
|
||||||
|
self.elasticsearch_client.index(
|
||||||
|
index=self.config.glossary_index_name,
|
||||||
|
id=str(glossary_doc.glossary_id),
|
||||||
|
body=glossary_doc.json(),
|
||||||
|
)
|
||||||
if isinstance(record, Topic):
|
if isinstance(record, Topic):
|
||||||
topic_doc = self._create_topic_es_doc(record)
|
topic_doc = self._create_topic_es_doc(record)
|
||||||
self.elasticsearch_client.index(
|
self.elasticsearch_client.index(
|
||||||
@ -443,6 +459,61 @@ class ElasticsearchSink(Sink[Entity]):
|
|||||||
|
|
||||||
return pipeline_doc
|
return pipeline_doc
|
||||||
|
|
||||||
|
def _create_glossary_es_doc(self, glossary: Glossary):
|
||||||
|
fqdn = glossary.fullyQualifiedName
|
||||||
|
suggest = [
|
||||||
|
{
|
||||||
|
"input": [
|
||||||
|
glossary.displayName if glossary.displayName else glossary.name
|
||||||
|
],
|
||||||
|
"weight": 10,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
tags = set()
|
||||||
|
timestamp = time.time()
|
||||||
|
glossary_owner = (
|
||||||
|
str(glossary.owner.id.__root__) if glossary.owner is not None else ""
|
||||||
|
)
|
||||||
|
glossary_followers = []
|
||||||
|
if glossary.followers:
|
||||||
|
for follower in glossary.followers.__root__:
|
||||||
|
glossary_followers.append(str(follower.id.__root__))
|
||||||
|
tier = None
|
||||||
|
for glossary_tag in glossary.tags:
|
||||||
|
if "Tier" in glossary_tag.tagFQN:
|
||||||
|
tier = glossary_tag.tagFQN
|
||||||
|
else:
|
||||||
|
tags.add(glossary_tag.tagFQN)
|
||||||
|
# tasks: List[Task] = glossary.tasks # TODO Handle Glossary words
|
||||||
|
# task_names = []
|
||||||
|
# task_descriptions = []
|
||||||
|
# for task in tasks:
|
||||||
|
# task_names.append(task.displayName)
|
||||||
|
# if task.description is not None:
|
||||||
|
# task_descriptions.append(task.description)
|
||||||
|
# if tags in task and len(task.tags) > 0:
|
||||||
|
# for col_tag in task.tags:
|
||||||
|
# tags.add(col_tag.tagFQN)
|
||||||
|
|
||||||
|
glossary_doc = GlossaryESDocument(
|
||||||
|
glossary_id=str(glossary.id.__root__),
|
||||||
|
glossary_name=glossary.displayName
|
||||||
|
if glossary.displayName
|
||||||
|
else glossary.name,
|
||||||
|
# task_names=task_names, # TODO Handle Glossary words
|
||||||
|
# task_descriptions=task_descriptions,
|
||||||
|
suggest=suggest,
|
||||||
|
description=glossary.description,
|
||||||
|
last_updated_timestamp=timestamp,
|
||||||
|
tier=tier,
|
||||||
|
tags=list(tags),
|
||||||
|
fqdn=fqdn,
|
||||||
|
owner=glossary_owner,
|
||||||
|
followers=glossary_followers,
|
||||||
|
)
|
||||||
|
|
||||||
|
return glossary_doc
|
||||||
|
|
||||||
def _get_charts(self, chart_refs: Optional[List[entityReference.EntityReference]]):
|
def _get_charts(self, chart_refs: Optional[List[entityReference.EntityReference]]):
|
||||||
charts = []
|
charts = []
|
||||||
if chart_refs:
|
if chart_refs:
|
||||||
|
@ -200,6 +200,48 @@ TOPIC_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
|
|||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
|
GLOSSARY_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
|
||||||
|
"""
|
||||||
|
{
|
||||||
|
"mappings":{
|
||||||
|
"properties": {
|
||||||
|
"glossary_name": {
|
||||||
|
"type":"text"
|
||||||
|
},
|
||||||
|
"display_name": {
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
|
"owner": {
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
|
"followers": {
|
||||||
|
"type": "keyword"
|
||||||
|
},
|
||||||
|
"last_updated_timestamp": {
|
||||||
|
"type": "date",
|
||||||
|
"format": "epoch_second"
|
||||||
|
},
|
||||||
|
"description": {
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
|
"tier": {
|
||||||
|
"type": "keyword"
|
||||||
|
},
|
||||||
|
"tags": {
|
||||||
|
"type": "keyword"
|
||||||
|
},
|
||||||
|
"entity_type": {
|
||||||
|
"type": "keyword"
|
||||||
|
},
|
||||||
|
"suggest": {
|
||||||
|
"type": "completion"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
|
DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
|
||||||
"""
|
"""
|
||||||
{
|
{
|
||||||
|
@ -23,6 +23,7 @@ from metadata.ingestion.api.common import Entity, WorkflowContext
|
|||||||
from metadata.ingestion.api.source import Source, SourceStatus
|
from metadata.ingestion.api.source import Source, SourceStatus
|
||||||
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
||||||
from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
|
from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
|
||||||
|
from metadata.ingestion.ometa.openmetadata_rest import Glossary
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@ -33,6 +34,7 @@ class MetadataTablesRestSourceConfig(ConfigModel):
|
|||||||
include_tables: Optional[bool] = True
|
include_tables: Optional[bool] = True
|
||||||
include_topics: Optional[bool] = True
|
include_topics: Optional[bool] = True
|
||||||
include_dashboards: Optional[bool] = True
|
include_dashboards: Optional[bool] = True
|
||||||
|
include_glossary: Optional[bool] = True
|
||||||
include_pipelines: Optional[bool] = True
|
include_pipelines: Optional[bool] = True
|
||||||
limit_records: int = 1000
|
limit_records: int = 1000
|
||||||
|
|
||||||
@ -60,6 +62,10 @@ class MetadataSourceStatus(SourceStatus):
|
|||||||
self.success.append(table_name)
|
self.success.append(table_name)
|
||||||
logger.info("Table Scanned: %s", table_name)
|
logger.info("Table Scanned: %s", table_name)
|
||||||
|
|
||||||
|
def scanned_glossary(self, glossary_name: str) -> None:
|
||||||
|
self.success.append(glossary_name)
|
||||||
|
logger.info("Glossary Scanned: {}".format(glossary_name))
|
||||||
|
|
||||||
def scanned_topic(self, topic_name: str) -> None:
|
def scanned_topic(self, topic_name: str) -> None:
|
||||||
"""scanned topic method
|
"""scanned topic method
|
||||||
|
|
||||||
@ -143,6 +149,7 @@ class MetadataSource(Source[Entity]):
|
|||||||
def next_record(self) -> Iterable[Entity]:
|
def next_record(self) -> Iterable[Entity]:
|
||||||
yield from self.fetch_table()
|
yield from self.fetch_table()
|
||||||
yield from self.fetch_topic()
|
yield from self.fetch_topic()
|
||||||
|
yield from self.fetch_glossary()
|
||||||
yield from self.fetch_dashboard()
|
yield from self.fetch_dashboard()
|
||||||
yield from self.fetch_pipeline()
|
yield from self.fetch_pipeline()
|
||||||
|
|
||||||
@ -247,6 +254,23 @@ class MetadataSource(Source[Entity]):
|
|||||||
break
|
break
|
||||||
after = pipeline_entities.after
|
after = pipeline_entities.after
|
||||||
|
|
||||||
|
def fetch_glossary(self) -> Glossary:
|
||||||
|
if self.config.include_glossary:
|
||||||
|
after = None
|
||||||
|
while True:
|
||||||
|
glossary_entities = self.metadata.list_entities(
|
||||||
|
entity=Glossary,
|
||||||
|
fields=["owner", "tags", "followers"],
|
||||||
|
after=after,
|
||||||
|
limit=self.config.limit_records,
|
||||||
|
)
|
||||||
|
for glossary in glossary_entities.entities:
|
||||||
|
self.status.scanned_glossary(glossary.name)
|
||||||
|
yield glossary
|
||||||
|
if glossary_entities.after is None:
|
||||||
|
break
|
||||||
|
after = glossary_entities.after
|
||||||
|
|
||||||
def get_status(self) -> SourceStatus:
|
def get_status(self) -> SourceStatus:
|
||||||
return self.status
|
return self.status
|
||||||
|
|
||||||
|
@ -27,6 +27,7 @@ from metadata.generated.schema.entity.data.mlmodel import MlModel
|
|||||||
from metadata.generated.schema.entity.data.pipeline import Pipeline
|
from metadata.generated.schema.entity.data.pipeline import Pipeline
|
||||||
from metadata.generated.schema.entity.data.report import Report
|
from metadata.generated.schema.entity.data.report import Report
|
||||||
from metadata.generated.schema.entity.data.table import Table
|
from metadata.generated.schema.entity.data.table import Table
|
||||||
|
from metadata.generated.schema.entity.data.glossary import Glossary
|
||||||
from metadata.generated.schema.entity.data.topic import Topic
|
from metadata.generated.schema.entity.data.topic import Topic
|
||||||
from metadata.generated.schema.entity.services.dashboardService import DashboardService
|
from metadata.generated.schema.entity.services.dashboardService import DashboardService
|
||||||
from metadata.generated.schema.entity.services.databaseService import DatabaseService
|
from metadata.generated.schema.entity.services.databaseService import DatabaseService
|
||||||
@ -51,6 +52,10 @@ class OMetaEndpointTest(TestCase):
|
|||||||
Pass Entities and test their suffix generation
|
Pass Entities and test their suffix generation
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# Glossary
|
||||||
|
self.assertEqual(self.metadata.get_suffix(Glossary), "/glossary")
|
||||||
|
|
||||||
# ML
|
# ML
|
||||||
self.assertEqual(self.metadata.get_suffix(MlModel), "/mlmodels")
|
self.assertEqual(self.metadata.get_suffix(MlModel), "/mlmodels")
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user