mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-06-27 04:22:05 +00:00
Squash merge glossary support
This commit is contained in:
sureshms
parent
52ec2e3ad1
commit
17adf5debc
13
bootstrap/sql/mysql/v004__create_db_connection_info.sql
Normal file
13
bootstrap/sql/mysql/v004__create_db_connection_info.sql
Normal file
@ -0,0 +1,13 @@
|
||||
CREATE TABLE IF NOT EXISTS glossary_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
|
||||
timestamp BIGINT,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY unique_name(fullyQualifiedName),
|
||||
INDEX (updatedBy),
|
||||
INDEX (updatedAt)
|
||||
);
|
||||
@ -70,6 +70,7 @@ public final class Entity {
|
||||
public static final String UNUSED = "unused";
|
||||
public static final String BOTS = "bots";
|
||||
public static final String LOCATION = "location";
|
||||
public static final String GLOSSARY = "glossary";
|
||||
|
||||
//
|
||||
// Policies
|
||||
|
||||
@ -12,6 +12,7 @@ import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
import javax.ws.rs.core.Response;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
@ -31,6 +32,7 @@ import org.elasticsearch.client.indices.PutMappingRequest;
|
||||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.openmetadata.catalog.Entity;
|
||||
import org.openmetadata.catalog.entity.data.Dashboard;
|
||||
import org.openmetadata.catalog.entity.data.Glossary;
|
||||
import org.openmetadata.catalog.entity.data.Pipeline;
|
||||
import org.openmetadata.catalog.entity.data.Table;
|
||||
import org.openmetadata.catalog.entity.data.Topic;
|
||||
@ -63,7 +65,8 @@ public class ElasticSearchIndexDefinition {
|
||||
TABLE_SEARCH_INDEX("table_search_index", "/elasticsearch/table_index_mapping.json"),
|
||||
TOPIC_SEARCH_INDEX("topic_search_index", "/elasticsearch/topic_index_mapping.json"),
|
||||
DASHBOARD_SEARCH_INDEX("dashboard_search_index", "/elasticsearch/dashboard_index_mapping.json"),
|
||||
PIPELINE_SEARCH_INDEX("pipeline_search_index", "/elasticsearch/pipeline_index_mapping.json");
|
||||
PIPELINE_SEARCH_INDEX("pipeline_search_index", "/elasticsearch/pipeline_index_mapping.json"),
|
||||
GLOSSARY_SEARCH_INDEX("glossary_search_index", "/elasticsearch/glossary_index_mapping.json");
|
||||
|
||||
public final String indexName;
|
||||
public final String indexMappingFile;
|
||||
@ -189,6 +192,8 @@ public class ElasticSearchIndexDefinition {
|
||||
return ElasticSearchIndexType.PIPELINE_SEARCH_INDEX;
|
||||
} else if (type.equalsIgnoreCase(Entity.TOPIC)) {
|
||||
return ElasticSearchIndexType.TOPIC_SEARCH_INDEX;
|
||||
} else if (type.equalsIgnoreCase(Entity.TOPIC)) {
|
||||
return ElasticSearchIndexType.GLOSSARY_SEARCH_INDEX;
|
||||
}
|
||||
throw new RuntimeException("Failed to find index doc for type " + type);
|
||||
}
|
||||
@ -707,3 +712,68 @@ class PipelineESIndex extends ElasticSearchIndex {
|
||||
return pipelineESIndexBuilder;
|
||||
}
|
||||
}
|
||||
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
@Getter
|
||||
@SuperBuilder(builderMethodName = "internalBuilder")
|
||||
@Value
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
class GlossaryESIndex extends ElasticSearchIndex {
|
||||
@JsonProperty("glossary_id")
|
||||
String glossaryId;
|
||||
|
||||
public static GlossaryESIndexBuilder builder(Glossary glossary, int responseCode) {
|
||||
List<String> tags = new ArrayList<>();
|
||||
List<String> taskNames = new ArrayList<>();
|
||||
List<String> taskDescriptions = new ArrayList<>();
|
||||
List<ElasticSearchSuggest> suggest = new ArrayList<>();
|
||||
suggest.add(ElasticSearchSuggest.builder().input(glossary.getFullyQualifiedName()).weight(5).build());
|
||||
suggest.add(ElasticSearchSuggest.builder().input(glossary.getDisplayName()).weight(10).build());
|
||||
|
||||
if (glossary.getTags() != null) {
|
||||
glossary.getTags().forEach(tag -> tags.add(tag.getTagFQN()));
|
||||
}
|
||||
|
||||
Long updatedTimestamp = glossary.getUpdatedAt();
|
||||
ParseTags parseTags = new ParseTags(tags);
|
||||
String description = glossary.getDescription() != null ? glossary.getDescription() : "";
|
||||
String displayName = glossary.getDisplayName() != null ? glossary.getDisplayName() : "";
|
||||
GlossaryESIndexBuilder builder =
|
||||
internalBuilder()
|
||||
.glossaryId(glossary.getId().toString())
|
||||
.name(glossary.getDisplayName())
|
||||
.displayName(description)
|
||||
.description(displayName)
|
||||
.fqdn(glossary.getFullyQualifiedName())
|
||||
.lastUpdatedTimestamp(updatedTimestamp)
|
||||
.entityType("glossary")
|
||||
.suggest(suggest)
|
||||
.tags(parseTags.tags)
|
||||
.tier(parseTags.tierTag);
|
||||
|
||||
if (glossary.getFollowers() != null) {
|
||||
builder.followers(
|
||||
glossary.getFollowers().stream().map(item -> item.getId().toString()).collect(Collectors.toList()));
|
||||
} else if (responseCode == Response.Status.CREATED.getStatusCode()) {
|
||||
builder.followers(Collections.emptyList());
|
||||
}
|
||||
|
||||
if (glossary.getOwner() != null) {
|
||||
builder.owner(glossary.getOwner().getId().toString());
|
||||
}
|
||||
|
||||
ESChangeDescription esChangeDescription = null;
|
||||
if (glossary.getChangeDescription() != null) {
|
||||
esChangeDescription =
|
||||
ESChangeDescription.builder().updatedAt(updatedTimestamp).updatedBy(glossary.getUpdatedBy()).build();
|
||||
esChangeDescription.setFieldsAdded(glossary.getChangeDescription().getFieldsAdded());
|
||||
esChangeDescription.setFieldsDeleted(glossary.getChangeDescription().getFieldsDeleted());
|
||||
esChangeDescription.setFieldsUpdated(glossary.getChangeDescription().getFieldsUpdated());
|
||||
} else if (responseCode == Response.Status.CREATED.getStatusCode()) {
|
||||
esChangeDescription =
|
||||
ESChangeDescription.builder().updatedAt(updatedTimestamp).updatedBy(glossary.getUpdatedBy()).build();
|
||||
}
|
||||
builder.changeDescriptions(esChangeDescription != null ? List.of(esChangeDescription) : null);
|
||||
return builder;
|
||||
}
|
||||
}
|
||||
|
||||
@ -33,6 +33,7 @@ import org.openmetadata.catalog.entity.Bots;
|
||||
import org.openmetadata.catalog.entity.data.Chart;
|
||||
import org.openmetadata.catalog.entity.data.Dashboard;
|
||||
import org.openmetadata.catalog.entity.data.Database;
|
||||
import org.openmetadata.catalog.entity.data.Glossary;
|
||||
import org.openmetadata.catalog.entity.data.Location;
|
||||
import org.openmetadata.catalog.entity.data.Metrics;
|
||||
import org.openmetadata.catalog.entity.data.MlModel;
|
||||
@ -58,6 +59,7 @@ import org.openmetadata.catalog.jdbi3.DashboardRepository.DashboardEntityInterfa
|
||||
import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface;
|
||||
import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface;
|
||||
import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
|
||||
import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface;
|
||||
import org.openmetadata.catalog.jdbi3.LocationRepository.LocationEntityInterface;
|
||||
import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface;
|
||||
import org.openmetadata.catalog.jdbi3.MetricsRepository.MetricsEntityInterface;
|
||||
@ -134,6 +136,9 @@ public interface CollectionDAO {
|
||||
@CreateSqlObject
|
||||
MlModelDAO mlModelDAO();
|
||||
|
||||
@CreateSqlObject
|
||||
GlossaryDAO glossaryDAO();
|
||||
|
||||
@CreateSqlObject
|
||||
BotsDAO botsDAO();
|
||||
|
||||
@ -689,6 +694,28 @@ public interface CollectionDAO {
|
||||
}
|
||||
}
|
||||
|
||||
interface GlossaryDAO extends EntityDAO<Glossary> {
|
||||
@Override
|
||||
default String getTableName() {
|
||||
return "glossary_entity";
|
||||
}
|
||||
|
||||
@Override
|
||||
default Class<Glossary> getEntityClass() {
|
||||
return Glossary.class;
|
||||
}
|
||||
|
||||
@Override
|
||||
default String getNameColumn() {
|
||||
return "fullyQualifiedName";
|
||||
}
|
||||
|
||||
@Override
|
||||
default EntityReference getEntityReference(Glossary entity) {
|
||||
return new GlossaryEntityInterface(entity).getEntityReference();
|
||||
}
|
||||
}
|
||||
|
||||
interface AirflowPipelineDAO extends EntityDAO<AirflowPipeline> {
|
||||
@Override
|
||||
default String getTableName() {
|
||||
|
||||
@ -0,0 +1,271 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.openmetadata.catalog.jdbi3;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.text.ParseException;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import org.jdbi.v3.sqlobject.transaction.Transaction;
|
||||
import org.openmetadata.catalog.Entity;
|
||||
import org.openmetadata.catalog.entity.data.Glossary;
|
||||
import org.openmetadata.catalog.resources.glossary.GlossaryResource;
|
||||
import org.openmetadata.catalog.type.ChangeDescription;
|
||||
import org.openmetadata.catalog.type.EntityReference;
|
||||
import org.openmetadata.catalog.type.TagLabel;
|
||||
import org.openmetadata.catalog.util.EntityInterface;
|
||||
import org.openmetadata.catalog.util.EntityUtil;
|
||||
import org.openmetadata.catalog.util.EntityUtil.Fields;
|
||||
import org.openmetadata.catalog.util.JsonUtils;
|
||||
|
||||
public class GlossaryRepository extends EntityRepository<Glossary> {
|
||||
private static final Fields GLOSSARY_UPDATE_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "owner,tags");
|
||||
private static final Fields GLOSSARY_PATCH_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "owner,tags");
|
||||
private final CollectionDAO dao;
|
||||
|
||||
public GlossaryRepository(CollectionDAO dao) {
|
||||
super(
|
||||
GlossaryResource.COLLECTION_PATH,
|
||||
Entity.GLOSSARY,
|
||||
Glossary.class,
|
||||
dao.glossaryDAO(),
|
||||
dao,
|
||||
GLOSSARY_PATCH_FIELDS,
|
||||
GLOSSARY_UPDATE_FIELDS,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
this.dao = dao;
|
||||
}
|
||||
|
||||
public static String getFQN(Glossary glossary) {
|
||||
return (glossary.getName());
|
||||
}
|
||||
|
||||
@Transaction
|
||||
public EntityReference getOwnerReference(Glossary glossary) throws IOException {
|
||||
return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), glossary.getOwner());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Glossary setFields(Glossary glossary, Fields fields) throws IOException, ParseException {
|
||||
glossary.setDisplayName(glossary.getDisplayName());
|
||||
glossary.setOwner(fields.contains("owner") ? getOwner(glossary) : null);
|
||||
glossary.setFollowers(fields.contains("followers") ? getFollowers(glossary) : null);
|
||||
glossary.setTags(fields.contains("tags") ? getTags(glossary.getFullyQualifiedName()) : null);
|
||||
glossary.setUsageSummary(
|
||||
fields.contains("usageSummary") ? EntityUtil.getLatestUsage(dao.usageDAO(), glossary.getId()) : null);
|
||||
return glossary;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void prepare(Glossary glossary) throws IOException {
|
||||
glossary.setFullyQualifiedName(getFQN(glossary));
|
||||
EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), glossary.getOwner()); // Validate owner
|
||||
glossary.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), glossary.getTags()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void storeEntity(Glossary glossary, boolean update) throws IOException {
|
||||
// Relationships and fields such as href are derived and not stored as part of json
|
||||
EntityReference owner = glossary.getOwner();
|
||||
List<TagLabel> tags = glossary.getTags();
|
||||
|
||||
// Don't store owner, dashboard, href and tags as JSON. Build it on the fly based on relationships
|
||||
glossary.withOwner(null).withHref(null).withTags(null);
|
||||
|
||||
if (update) {
|
||||
dao.glossaryDAO().update(glossary.getId(), JsonUtils.pojoToJson(glossary));
|
||||
} else {
|
||||
dao.glossaryDAO().insert(glossary);
|
||||
}
|
||||
|
||||
// Restore the relationships
|
||||
glossary.withOwner(owner).withTags(tags);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void restorePatchAttributes(Glossary original, Glossary updated) {}
|
||||
|
||||
@Override
|
||||
public EntityInterface<Glossary> getEntityInterface(Glossary entity) {
|
||||
return new GlossaryEntityInterface(entity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void storeRelationships(Glossary glossary) {
|
||||
setOwner(glossary, glossary.getOwner());
|
||||
applyTags(glossary);
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntityUpdater getUpdater(Glossary original, Glossary updated, Operation operation) {
|
||||
return new GlossaryUpdater(original, updated, operation);
|
||||
}
|
||||
|
||||
public static class GlossaryEntityInterface implements EntityInterface<Glossary> {
|
||||
private final Glossary entity;
|
||||
|
||||
public GlossaryEntityInterface(Glossary entity) {
|
||||
this.entity = entity;
|
||||
}
|
||||
|
||||
@Override
|
||||
public UUID getId() {
|
||||
return entity.getId();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return entity.getDescription();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDisplayName() {
|
||||
return entity.getDisplayName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Boolean isDeleted() {
|
||||
return entity.getDeleted();
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntityReference getOwner() {
|
||||
return entity.getOwner();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFullyQualifiedName() {
|
||||
return entity.getFullyQualifiedName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<TagLabel> getTags() {
|
||||
return entity.getTags();
|
||||
}
|
||||
|
||||
public String getSkos() {
|
||||
return entity.getSkos();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Double getVersion() {
|
||||
return entity.getVersion();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUpdatedBy() {
|
||||
return entity.getUpdatedBy();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getUpdatedAt() {
|
||||
return entity.getUpdatedAt();
|
||||
}
|
||||
|
||||
@Override
|
||||
public URI getHref() {
|
||||
return entity.getHref();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<EntityReference> getFollowers() {
|
||||
return entity.getFollowers();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChangeDescription getChangeDescription() {
|
||||
return entity.getChangeDescription();
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntityReference getEntityReference() {
|
||||
return new EntityReference()
|
||||
.withId(getId())
|
||||
.withName(getFullyQualifiedName())
|
||||
.withDescription(getDescription())
|
||||
.withDisplayName(getDisplayName())
|
||||
.withType(Entity.GLOSSARY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Glossary getEntity() {
|
||||
return entity;
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntityReference getContainer() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setId(UUID id) {
|
||||
entity.setId(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setDescription(String description) {
|
||||
entity.setDescription(description);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setDisplayName(String displayName) {
|
||||
entity.setDisplayName(displayName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setUpdateDetails(String updatedBy, long updatedAt) {
|
||||
entity.setUpdatedBy(updatedBy);
|
||||
entity.setUpdatedAt(updatedAt);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setChangeDescription(Double newVersion, ChangeDescription changeDescription) {
|
||||
entity.setVersion(newVersion);
|
||||
entity.setChangeDescription(changeDescription);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setOwner(EntityReference owner) {
|
||||
entity.setOwner(owner);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setDeleted(boolean flag) {
|
||||
entity.setDeleted(flag);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Glossary withHref(URI href) {
|
||||
return entity.withHref(href);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setTags(List<TagLabel> tags) {
|
||||
entity.setTags(tags);
|
||||
}
|
||||
}
|
||||
|
||||
/** Handles entity updated from PUT and POST operation. */
|
||||
public class GlossaryUpdater extends EntityUpdater {
|
||||
public GlossaryUpdater(Glossary original, Glossary updated, Operation operation) {
|
||||
super(original, updated, operation);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,436 @@
|
||||
/*
|
||||
* Copyright 2021 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.openmetadata.catalog.resources.glossary;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
import io.swagger.annotations.Api;
|
||||
import io.swagger.v3.oas.annotations.ExternalDocumentation;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.media.Content;
|
||||
import io.swagger.v3.oas.annotations.media.ExampleObject;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import io.swagger.v3.oas.annotations.parameters.RequestBody;
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponse;
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.security.GeneralSecurityException;
|
||||
import java.text.ParseException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import javax.json.JsonPatch;
|
||||
import javax.validation.Valid;
|
||||
import javax.validation.constraints.Max;
|
||||
import javax.validation.constraints.Min;
|
||||
import javax.ws.rs.Consumes;
|
||||
import javax.ws.rs.DELETE;
|
||||
import javax.ws.rs.DefaultValue;
|
||||
import javax.ws.rs.GET;
|
||||
import javax.ws.rs.PATCH;
|
||||
import javax.ws.rs.POST;
|
||||
import javax.ws.rs.PUT;
|
||||
import javax.ws.rs.Path;
|
||||
import javax.ws.rs.PathParam;
|
||||
import javax.ws.rs.Produces;
|
||||
import javax.ws.rs.QueryParam;
|
||||
import javax.ws.rs.core.Context;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import javax.ws.rs.core.Response;
|
||||
import javax.ws.rs.core.SecurityContext;
|
||||
import javax.ws.rs.core.UriInfo;
|
||||
import org.openmetadata.catalog.Entity;
|
||||
import org.openmetadata.catalog.api.data.CreateGlossary;
|
||||
import org.openmetadata.catalog.entity.data.Glossary;
|
||||
import org.openmetadata.catalog.jdbi3.CollectionDAO;
|
||||
import org.openmetadata.catalog.jdbi3.GlossaryRepository;
|
||||
import org.openmetadata.catalog.resources.Collection;
|
||||
import org.openmetadata.catalog.security.Authorizer;
|
||||
import org.openmetadata.catalog.security.SecurityUtil;
|
||||
import org.openmetadata.catalog.type.EntityHistory;
|
||||
import org.openmetadata.catalog.type.EntityReference;
|
||||
import org.openmetadata.catalog.type.Include;
|
||||
import org.openmetadata.catalog.util.EntityUtil.Fields;
|
||||
import org.openmetadata.catalog.util.RestUtil;
|
||||
import org.openmetadata.catalog.util.RestUtil.DeleteResponse;
|
||||
import org.openmetadata.catalog.util.RestUtil.PatchResponse;
|
||||
import org.openmetadata.catalog.util.RestUtil.PutResponse;
|
||||
import org.openmetadata.catalog.util.ResultList;
|
||||
|
||||
@Path("/v1/glossary")
|
||||
@Api(value = "Glossary collection", tags = "Glossary collection")
|
||||
@Produces(MediaType.APPLICATION_JSON)
|
||||
@Consumes(MediaType.APPLICATION_JSON)
|
||||
@Collection(name = "glossary")
|
||||
public class GlossaryResource {
|
||||
public static final String COLLECTION_PATH = "v1/glossary/";
|
||||
private final GlossaryRepository dao;
|
||||
private final Authorizer authorizer;
|
||||
|
||||
public static void addHref(UriInfo uriInfo, EntityReference ref) {
|
||||
ref.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, ref.getId()));
|
||||
}
|
||||
|
||||
public static List<Glossary> addHref(UriInfo uriInfo, List<Glossary> glossary) {
|
||||
Optional.ofNullable(glossary).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
|
||||
return glossary;
|
||||
}
|
||||
|
||||
public static Glossary addHref(UriInfo uriInfo, Glossary glossary) {
|
||||
glossary.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, glossary.getId()));
|
||||
Entity.withHref(uriInfo, glossary.getOwner());
|
||||
Entity.withHref(uriInfo, glossary.getFollowers());
|
||||
return glossary;
|
||||
}
|
||||
|
||||
@Inject
|
||||
public GlossaryResource(CollectionDAO dao, Authorizer authorizer) {
|
||||
Objects.requireNonNull(dao, "GlossaryRepository must not be null");
|
||||
this.dao = new GlossaryRepository(dao);
|
||||
this.authorizer = authorizer;
|
||||
}
|
||||
|
||||
public static class GlossaryList extends ResultList<Glossary> {
|
||||
@SuppressWarnings("unused")
|
||||
GlossaryList() {
|
||||
// Empty constructor needed for deserialization
|
||||
}
|
||||
|
||||
public GlossaryList(List<Glossary> data, String beforeCursor, String afterCursor, int total)
|
||||
throws GeneralSecurityException, UnsupportedEncodingException {
|
||||
super(data, beforeCursor, afterCursor, total);
|
||||
}
|
||||
}
|
||||
|
||||
static final String FIELDS = "owner,dashboard,definition,followers,tags,usageSummary,skos";
|
||||
public static final List<String> FIELD_LIST = Arrays.asList(FIELDS.replaceAll(" ", "").split(","));
|
||||
|
||||
@GET
|
||||
@Valid
|
||||
@Operation(
|
||||
summary = "List Glossary",
|
||||
tags = "glossary",
|
||||
description =
|
||||
"Get a list of glossary. Use `fields` parameter to get only necessary fields. "
|
||||
+ " Use cursor-based pagination to limit the number "
|
||||
+ "entries in the list using `limit` and `before` or `after` query params.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "List of glossary",
|
||||
content = @Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryList.class)))
|
||||
})
|
||||
public ResultList<Glossary> list(
|
||||
@Context UriInfo uriInfo,
|
||||
@Context SecurityContext securityContext,
|
||||
@Parameter(
|
||||
description = "Fields requested in the returned resource",
|
||||
schema = @Schema(type = "string", example = FIELDS))
|
||||
@QueryParam("fields")
|
||||
String fieldsParam,
|
||||
@Parameter(description = "Limit the number glossary returned. (1 to 1000000, " + "default = 10)")
|
||||
@DefaultValue("10")
|
||||
@Min(1)
|
||||
@Max(1000000)
|
||||
@QueryParam("limit")
|
||||
int limitParam,
|
||||
@Parameter(description = "Returns list of glossary before this cursor", schema = @Schema(type = "string"))
|
||||
@QueryParam("before")
|
||||
String before,
|
||||
@Parameter(description = "Returns list of glossary after this cursor", schema = @Schema(type = "string"))
|
||||
@QueryParam("after")
|
||||
String after,
|
||||
@Parameter(
|
||||
description = "Include all, deleted, or non-deleted entities.",
|
||||
schema = @Schema(implementation = Include.class))
|
||||
@QueryParam("include")
|
||||
@DefaultValue("non-deleted")
|
||||
Include include)
|
||||
throws IOException, GeneralSecurityException, ParseException {
|
||||
RestUtil.validateCursors(before, after);
|
||||
Fields fields = new Fields(FIELD_LIST, fieldsParam);
|
||||
|
||||
ResultList<Glossary> glossary;
|
||||
if (before != null) { // Reverse paging
|
||||
glossary = dao.listBefore(uriInfo, fields, null, limitParam, before, include); // Ask for one extra entry
|
||||
} else { // Forward paging or first page
|
||||
glossary = dao.listAfter(uriInfo, fields, null, limitParam, after, include);
|
||||
}
|
||||
addHref(uriInfo, glossary.getData());
|
||||
return glossary;
|
||||
}
|
||||
|
||||
@GET
|
||||
@Path("/{id}")
|
||||
@Operation(
|
||||
summary = "Get a glossary",
|
||||
tags = "glossary",
|
||||
description = "Get a glossary by `id`.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "The glossary",
|
||||
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
|
||||
@ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found")
|
||||
})
|
||||
public Glossary get(
|
||||
@Context UriInfo uriInfo,
|
||||
@Context SecurityContext securityContext,
|
||||
@PathParam("id") String id,
|
||||
@Parameter(
|
||||
description = "Fields requested in the returned resource",
|
||||
schema = @Schema(type = "string", example = FIELDS))
|
||||
@QueryParam("fields")
|
||||
String fieldsParam,
|
||||
@Parameter(
|
||||
description = "Include all, deleted, or non-deleted entities.",
|
||||
schema = @Schema(implementation = Include.class))
|
||||
@QueryParam("include")
|
||||
@DefaultValue("non-deleted")
|
||||
Include include)
|
||||
throws IOException, ParseException {
|
||||
Fields fields = new Fields(FIELD_LIST, fieldsParam);
|
||||
return addHref(uriInfo, dao.get(uriInfo, id, fields, include));
|
||||
}
|
||||
|
||||
@GET
|
||||
@Path("/name/{fqn}")
|
||||
@Operation(
|
||||
summary = "Get a glossary by name",
|
||||
tags = "glossary",
|
||||
description = "Get a glossary by fully qualified name.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "The glossary",
|
||||
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
|
||||
@ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found")
|
||||
})
|
||||
public Glossary getByName(
|
||||
@Context UriInfo uriInfo,
|
||||
@PathParam("fqn") String fqn,
|
||||
@Context SecurityContext securityContext,
|
||||
@Parameter(
|
||||
description = "Fields requested in the returned resource",
|
||||
schema = @Schema(type = "string", example = FIELDS))
|
||||
@QueryParam("fields")
|
||||
String fieldsParam,
|
||||
@Parameter(
|
||||
description = "Include all, deleted, or non-deleted entities.",
|
||||
schema = @Schema(implementation = Include.class))
|
||||
@QueryParam("include")
|
||||
@DefaultValue("non-deleted")
|
||||
Include include)
|
||||
throws IOException, ParseException {
|
||||
Fields fields = new Fields(FIELD_LIST, fieldsParam);
|
||||
Glossary glossary = dao.getByName(uriInfo, fqn, fields, include);
|
||||
return addHref(uriInfo, glossary);
|
||||
}
|
||||
|
||||
@GET
|
||||
@Path("/{id}/versions")
|
||||
@Operation(
|
||||
summary = "List glossary versions",
|
||||
tags = "glossary",
|
||||
description = "Get a list of all the versions of a glossary identified by `id`",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "List of glossary versions",
|
||||
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class)))
|
||||
})
|
||||
public EntityHistory listVersions(
|
||||
@Context UriInfo uriInfo,
|
||||
@Context SecurityContext securityContext,
|
||||
@Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id)
|
||||
throws IOException, ParseException {
|
||||
return dao.listVersions(id);
|
||||
}
|
||||
|
||||
@GET
|
||||
@Path("/{id}/versions/{version}")
|
||||
@Operation(
|
||||
summary = "Get a version of the glossary",
|
||||
tags = "glossary",
|
||||
description = "Get a version of the glossary by given `id`",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "glossary",
|
||||
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
|
||||
@ApiResponse(
|
||||
responseCode = "404",
|
||||
description = "Glossary for instance {id} and version {version} is " + "not found")
|
||||
})
|
||||
public Glossary getVersion(
|
||||
@Context UriInfo uriInfo,
|
||||
@Context SecurityContext securityContext,
|
||||
@Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id,
|
||||
@Parameter(
|
||||
description = "glossary version number in the form `major`.`minor`",
|
||||
schema = @Schema(type = "string", example = "0.1 or 1.1"))
|
||||
@PathParam("version")
|
||||
String version)
|
||||
throws IOException, ParseException {
|
||||
return dao.getVersion(id, version);
|
||||
}
|
||||
|
||||
@POST
|
||||
@Operation(
|
||||
summary = "Create a glossary",
|
||||
tags = "glossary",
|
||||
description = "Create a new glossary.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "The glossary",
|
||||
content =
|
||||
@Content(mediaType = "application/json", schema = @Schema(implementation = CreateGlossary.class))),
|
||||
@ApiResponse(responseCode = "400", description = "Bad request")
|
||||
})
|
||||
public Response create(
|
||||
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create)
|
||||
throws IOException, ParseException {
|
||||
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
|
||||
Glossary glossary = getGlossary(securityContext, create);
|
||||
glossary = addHref(uriInfo, dao.create(uriInfo, glossary));
|
||||
return Response.created(glossary.getHref()).entity(glossary).build();
|
||||
}
|
||||
|
||||
@PATCH
|
||||
@Path("/{id}")
|
||||
@Operation(
|
||||
summary = "Update a glossary",
|
||||
tags = "glossary",
|
||||
description = "Update an existing glossary using JsonPatch.",
|
||||
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
|
||||
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
|
||||
public Response updateDescription(
|
||||
@Context UriInfo uriInfo,
|
||||
@Context SecurityContext securityContext,
|
||||
@PathParam("id") String id,
|
||||
@RequestBody(
|
||||
description = "JsonPatch with array of operations",
|
||||
content =
|
||||
@Content(
|
||||
mediaType = MediaType.APPLICATION_JSON_PATCH_JSON,
|
||||
examples = {
|
||||
@ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]")
|
||||
}))
|
||||
JsonPatch patch)
|
||||
throws IOException, ParseException {
|
||||
Fields fields = new Fields(FIELD_LIST, FIELDS);
|
||||
Glossary glossary = dao.get(uriInfo, id, fields);
|
||||
SecurityUtil.checkAdminRoleOrPermissions(
|
||||
authorizer, securityContext, dao.getEntityInterface(glossary).getEntityReference(), patch);
|
||||
PatchResponse<Glossary> response =
|
||||
dao.patch(uriInfo, UUID.fromString(id), securityContext.getUserPrincipal().getName(), patch);
|
||||
addHref(uriInfo, response.getEntity());
|
||||
return response.toResponse();
|
||||
}
|
||||
|
||||
@PUT
|
||||
@Operation(
|
||||
summary = "Create or update a glossary",
|
||||
tags = "glossary",
|
||||
description = "Create a new glossary, if it does not exist or update an existing glossary.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "The glossary",
|
||||
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
|
||||
@ApiResponse(responseCode = "400", description = "Bad request")
|
||||
})
|
||||
public Response createOrUpdate(
|
||||
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create)
|
||||
throws IOException, ParseException {
|
||||
Glossary glossary = getGlossary(securityContext, create);
|
||||
PutResponse<Glossary> response = dao.createOrUpdate(uriInfo, glossary);
|
||||
addHref(uriInfo, response.getEntity());
|
||||
return response.toResponse();
|
||||
}
|
||||
|
||||
@PUT
|
||||
@Path("/{id}/followers")
|
||||
@Operation(
|
||||
summary = "Add a follower",
|
||||
tags = "glossary",
|
||||
description = "Add a user identified by `userId` as follower of this glossary",
|
||||
responses = {
|
||||
@ApiResponse(responseCode = "200", description = "OK"),
|
||||
@ApiResponse(responseCode = "404", description = "glossary for instance {id} is not found")
|
||||
})
|
||||
public Response addFollower(
|
||||
@Context UriInfo uriInfo,
|
||||
@Context SecurityContext securityContext,
|
||||
@Parameter(description = "Id of the glossary", schema = @Schema(type = "string")) @PathParam("id") String id,
|
||||
@Parameter(description = "Id of the user to be added as follower", schema = @Schema(type = "string"))
|
||||
String userId)
|
||||
throws IOException, ParseException {
|
||||
return dao.addFollower(securityContext.getUserPrincipal().getName(), UUID.fromString(id), UUID.fromString(userId))
|
||||
.toResponse();
|
||||
}
|
||||
|
||||
@DELETE
|
||||
@Path("/{id}/followers/{userId}")
|
||||
@Operation(
|
||||
summary = "Remove a follower",
|
||||
tags = "glossary",
|
||||
description = "Remove the user identified `userId` as a follower of the glossary.")
|
||||
public Response deleteFollower(
|
||||
@Context UriInfo uriInfo,
|
||||
@Context SecurityContext securityContext,
|
||||
@Parameter(description = "Id of the glossary", schema = @Schema(type = "string")) @PathParam("id") String id,
|
||||
@Parameter(description = "Id of the user being removed as follower", schema = @Schema(type = "string"))
|
||||
@PathParam("userId")
|
||||
String userId)
|
||||
throws IOException, ParseException {
|
||||
return dao.deleteFollower(
|
||||
securityContext.getUserPrincipal().getName(), UUID.fromString(id), UUID.fromString(userId))
|
||||
.toResponse();
|
||||
}
|
||||
|
||||
@DELETE
|
||||
@Path("/{id}")
|
||||
@Operation(
|
||||
summary = "Delete a Glossary",
|
||||
tags = "glossary",
|
||||
description = "Delete a glossary by `id`.",
|
||||
responses = {
|
||||
@ApiResponse(responseCode = "200", description = "OK"),
|
||||
@ApiResponse(responseCode = "404", description = "glossary for instance {id} is not found")
|
||||
})
|
||||
public Response delete(@Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("id") String id)
|
||||
throws IOException, ParseException {
|
||||
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
|
||||
DeleteResponse<Glossary> response = dao.delete(securityContext.getUserPrincipal().getName(), id);
|
||||
return response.toResponse();
|
||||
}
|
||||
|
||||
private Glossary getGlossary(SecurityContext securityContext, CreateGlossary create) {
|
||||
return new Glossary()
|
||||
.withId(UUID.randomUUID())
|
||||
.withName(create.getName())
|
||||
.withDisplayName(create.getDisplayName())
|
||||
.withDescription(create.getDescription())
|
||||
.withSkos(create.getSkos())
|
||||
.withTags(create.getTags())
|
||||
.withOwner(create.getOwner())
|
||||
.withUpdatedBy(securityContext.getUserPrincipal().getName())
|
||||
.withUpdatedAt(System.currentTimeMillis());
|
||||
}
|
||||
}
|
||||
@ -146,6 +146,9 @@ public class SearchResource {
|
||||
case "table_search_index":
|
||||
searchSourceBuilder = buildTableSearchBuilder(query, from, size);
|
||||
break;
|
||||
case "glossary_search_index":
|
||||
searchSourceBuilder = buildGlossarySearchBuilder(query, from, size);
|
||||
break;
|
||||
default:
|
||||
searchSourceBuilder = buildAggregateSearchBuilder(query, from, size);
|
||||
break;
|
||||
@ -353,4 +356,27 @@ public class SearchResource {
|
||||
|
||||
return searchSourceBuilder;
|
||||
}
|
||||
|
||||
private SearchSourceBuilder buildGlossarySearchBuilder(String query, int from, int size) {
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
HighlightBuilder.Field highlightGlossaryName = new HighlightBuilder.Field("glossary_name");
|
||||
highlightGlossaryName.highlighterType("unified");
|
||||
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field("description");
|
||||
highlightDescription.highlighterType("unified");
|
||||
HighlightBuilder hb = new HighlightBuilder();
|
||||
hb.field(highlightDescription);
|
||||
hb.field(highlightGlossaryName);
|
||||
hb.preTags("<span class=\"text-highlighter\">");
|
||||
hb.postTags("</span>");
|
||||
searchSourceBuilder
|
||||
.query(QueryBuilders.queryStringQuery(query).field("glossary_name", 5.0f).field("description").lenient(true))
|
||||
.aggregation(AggregationBuilders.terms("EntityType").field("entity_type"))
|
||||
.aggregation(AggregationBuilders.terms("Tier").field("tier"))
|
||||
.aggregation(AggregationBuilders.terms("Tags").field("tags"))
|
||||
.highlighter(hb)
|
||||
.from(from)
|
||||
.size(size);
|
||||
|
||||
return searchSourceBuilder;
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,40 @@
|
||||
{
|
||||
"$id": "https://open-metadata.org/schema/api/data/createGlossary.json",
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Create Glossary entity request",
|
||||
"description": "Create Glossary entity request",
|
||||
"type": "object",
|
||||
"properties" : {
|
||||
"name": {
|
||||
"description": "Name that identifies this glossary.",
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 128
|
||||
},
|
||||
"displayName": {
|
||||
"description": "Display Name that identifies this glossary.",
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"description": "Description of the glossary instance.",
|
||||
"type": "string"
|
||||
},
|
||||
"skos": {
|
||||
"description": "SKOS data in JSON-LD format",
|
||||
"type": "string"
|
||||
},
|
||||
"tags": {
|
||||
"description": "Tags for this glossary",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "../../type/tagLabel.json"
|
||||
},
|
||||
"default": null
|
||||
},
|
||||
"owner": {
|
||||
"description": "Owner of this glossary",
|
||||
"$ref": "../../type/entityReference.json"
|
||||
}
|
||||
},
|
||||
"required": ["name"]
|
||||
}
|
||||
@ -0,0 +1,85 @@
|
||||
{
|
||||
"$id": "https://open-metadata.org/schema/entity/data/glossary.json",
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Glossary",
|
||||
"description": "This schema defines the Glossary entity based on SKOS.",
|
||||
"type": "object",
|
||||
|
||||
"properties" : {
|
||||
"id": {
|
||||
"description": "Unique identifier of a glossary instance.",
|
||||
"$ref": "../../type/basic.json#/definitions/uuid"
|
||||
},
|
||||
"name": {
|
||||
"description": "Name that identifies this glossary.",
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 128
|
||||
},
|
||||
"fullyQualifiedName": {
|
||||
"description": "A unique name that identifies a glossary.",
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 256
|
||||
},
|
||||
"displayName": {
|
||||
"description": "Display Name that identifies this glossary.",
|
||||
"type": "string"
|
||||
},
|
||||
"version" : {
|
||||
"description": "Metadata version of the entity.",
|
||||
"$ref": "../../type/entityHistory.json#/definitions/entityVersion"
|
||||
},
|
||||
"updatedAt" : {
|
||||
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
|
||||
"$ref": "../../type/basic.json#/definitions/timestamp"
|
||||
},
|
||||
"updatedBy" : {
|
||||
"description": "User who made the update.",
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"description": "Description of the glossary.",
|
||||
"type": "string"
|
||||
},
|
||||
"skos": {
|
||||
"description": "SKOS data in JSON-LD format",
|
||||
"type": "string"
|
||||
},
|
||||
"href": {
|
||||
"description": "Link to the resource corresponding to this entity.",
|
||||
"$ref": "../../type/basic.json#/definitions/href"
|
||||
},
|
||||
"owner": {
|
||||
"description": "Owner of this glossary.",
|
||||
"$ref": "../../type/entityReference.json"
|
||||
},
|
||||
"followers": {
|
||||
"description": "Followers of this glossary.",
|
||||
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList"
|
||||
},
|
||||
"tags": {
|
||||
"description": "Tags for this glossary.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "../../type/tagLabel.json"
|
||||
},
|
||||
"default": null
|
||||
},
|
||||
"usageSummary" : {
|
||||
"description": "Latest usage information for this glossary.",
|
||||
"$ref": "../../type/usageDetails.json",
|
||||
"default": null
|
||||
},
|
||||
"changeDescription": {
|
||||
"description" : "Change that lead to this version of the entity.",
|
||||
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
|
||||
},
|
||||
"deleted" : {
|
||||
"description": "When `true` indicates the entity has been soft deleted.",
|
||||
"type" : "boolean",
|
||||
"default": false
|
||||
}
|
||||
},
|
||||
"required": ["id", "name"]
|
||||
}
|
||||
@ -0,0 +1,259 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.openmetadata.catalog.resources.glossary;
|
||||
|
||||
import static javax.ws.rs.core.Response.Status.FORBIDDEN;
|
||||
import static javax.ws.rs.core.Response.Status.NOT_FOUND;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.openmetadata.catalog.security.SecurityUtil.authHeaders;
|
||||
import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
|
||||
import static org.openmetadata.catalog.util.TestUtils.NON_EXISTENT_ENTITY;
|
||||
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import org.apache.http.client.HttpResponseException;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.MethodOrderer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInfo;
|
||||
import org.junit.jupiter.api.TestMethodOrder;
|
||||
import org.openmetadata.catalog.CatalogApplicationTest;
|
||||
import org.openmetadata.catalog.Entity;
|
||||
import org.openmetadata.catalog.api.data.CreateGlossary;
|
||||
import org.openmetadata.catalog.entity.data.Glossary;
|
||||
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
|
||||
import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface;
|
||||
import org.openmetadata.catalog.resources.EntityResourceTest;
|
||||
import org.openmetadata.catalog.type.EntityReference;
|
||||
import org.openmetadata.catalog.util.TestUtils;
|
||||
|
||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||
public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlossary> {
|
||||
public GlossaryResourceTest() {
|
||||
super(
|
||||
Entity.GLOSSARY,
|
||||
Glossary.class,
|
||||
GlossaryResource.GlossaryList.class,
|
||||
"glossary",
|
||||
GlossaryResource.FIELDS,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true);
|
||||
}
|
||||
|
||||
@BeforeAll
|
||||
public void setup(TestInfo test) throws IOException, URISyntaxException {
|
||||
super.setup(test);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CreateGlossary createRequest(String name, String description, String displayName, EntityReference owner) {
|
||||
return new CreateGlossary()
|
||||
.withName(name)
|
||||
.withDescription(description)
|
||||
.withDisplayName(displayName)
|
||||
.withOwner(owner);
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntityReference getContainer(CreateGlossary createRequest) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// TODO: Entity tests
|
||||
|
||||
// @Test
|
||||
// public void patch_entityAttributes_200_ok()
|
||||
// {}
|
||||
|
||||
// @Test
|
||||
// public void put_entityCreate_200()
|
||||
// {}
|
||||
|
||||
// @Test
|
||||
// public void put_entityCreate_as_owner_200()
|
||||
// {}
|
||||
|
||||
// @Test
|
||||
// public void put_entityEmptyDescriptionUpdate_200()
|
||||
// {}
|
||||
|
||||
// @Test
|
||||
// public void put_entityNullDescriptionUpdate_200()
|
||||
// {}
|
||||
|
||||
// @Test
|
||||
// public void put_entityUpdateOwner_200()
|
||||
// {}
|
||||
|
||||
// @Test
|
||||
// public void put_entityUpdateWithNoChange_200()
|
||||
// {}
|
||||
|
||||
@Test
|
||||
public void post_validGlossary_200_OK(TestInfo test) throws IOException {
|
||||
CreateGlossary create = create(test).withDescription("description");
|
||||
createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void post_glossaryWithUserOwner_200_ok(TestInfo test) throws IOException {
|
||||
createAndCheckEntity(create(test).withOwner(USER_OWNER1), ADMIN_AUTH_HEADERS);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void post_glossaryWithTeamOwner_200_ok(TestInfo test) throws IOException {
|
||||
createAndCheckEntity(create(test).withOwner(TEAM_OWNER1), ADMIN_AUTH_HEADERS);
|
||||
}
|
||||
|
||||
// @Test
|
||||
// public void post_glossaryWithInvalidOwnerType_4xx(TestInfo test) {
|
||||
// EntityReference owner = new EntityReference().withId(TEAM1.getId()); /* No owner type is set */
|
||||
// CreateGlossary create = create(test).withOwner(owner);
|
||||
// HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
|
||||
// createEntity(create, ADMIN_AUTH_HEADERS));
|
||||
// TestUtils.assertResponseContains(exception, BAD_REQUEST, "type must not be null");
|
||||
// }
|
||||
|
||||
@Test
|
||||
public void post_glossary_as_non_admin_401(TestInfo test) {
|
||||
CreateGlossary create = create(test);
|
||||
HttpResponseException exception =
|
||||
assertThrows(HttpResponseException.class, () -> createEntity(create, authHeaders("test@open-metadata.org")));
|
||||
assertResponse(exception, FORBIDDEN, "Principal: CatalogPrincipal{name='test'} is not admin");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void get_nonExistentGlossary_404_notFound() {
|
||||
HttpResponseException exception =
|
||||
assertThrows(HttpResponseException.class, () -> getEntity(NON_EXISTENT_ENTITY, ADMIN_AUTH_HEADERS));
|
||||
assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, NON_EXISTENT_ENTITY));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void delete_glossary_200_ok(TestInfo test) throws HttpResponseException {
|
||||
Glossary glossary = createEntity(create(test), ADMIN_AUTH_HEADERS);
|
||||
deleteGlossary(glossary.getId(), ADMIN_AUTH_HEADERS);
|
||||
}
|
||||
|
||||
// @Test
|
||||
// public void delete_glossary_as_non_admin_401(TestInfo test) throws HttpResponseException {
|
||||
// Glossary glossary = createEntity(create(test), ADMIN_AUTH_HEADERS);
|
||||
// HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
|
||||
// deleteGlossary(glossary.getId(), authHeaders("test@open-metadata.org")));
|
||||
// assertResponse(exception, FORBIDDEN, "Principal: CatalogPrincipal{name='test'} is not admin");
|
||||
// }
|
||||
|
||||
@Test
|
||||
public void delete_nonExistentGlossary_404() {
|
||||
HttpResponseException exception =
|
||||
assertThrows(HttpResponseException.class, () -> getEntity(NON_EXISTENT_ENTITY, ADMIN_AUTH_HEADERS));
|
||||
assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, NON_EXISTENT_ENTITY));
|
||||
}
|
||||
|
||||
// @Test
|
||||
// public void put_GlossaryCreate_200(TestInfo test) throws HttpResponseException {
|
||||
// // Create a new Glossary with PUT
|
||||
// CreateGlossary request = create(test).withOwner(USER_OWNER1);
|
||||
// updateAndCheckGlossary(null, request.withName(test.getDisplayName()).withDescription(null), CREATED,
|
||||
// ADMIN_AUTH_HEADERS, NO_CHANGE);
|
||||
// }
|
||||
|
||||
public static CreateGlossary create(TestInfo test) {
|
||||
return create(test, 0);
|
||||
}
|
||||
|
||||
public static CreateGlossary create(TestInfo test, int index) {
|
||||
return new CreateGlossary().withName(getGlossaryName(test, index));
|
||||
}
|
||||
|
||||
/**
|
||||
* A method variant to be called form other tests to create a glossary without depending on Database, DatabaseService
|
||||
* set up in the {@code setup()} method
|
||||
*/
|
||||
public Glossary createEntity(TestInfo test, int index) throws IOException {
|
||||
CreateGlossary create = new CreateGlossary().withName(getGlossaryName(test, index));
|
||||
return createEntity(create, ADMIN_AUTH_HEADERS);
|
||||
}
|
||||
|
||||
private void deleteGlossary(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
|
||||
TestUtils.delete(CatalogApplicationTest.getResource("glossary/" + id), authHeaders);
|
||||
|
||||
// Check to make sure database entity does not exist
|
||||
HttpResponseException exception = assertThrows(HttpResponseException.class, () -> getEntity(id, authHeaders));
|
||||
assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, id));
|
||||
}
|
||||
|
||||
public static String getGlossaryName(TestInfo test, int index) {
|
||||
return String.format("glossary%d_%s", index, test.getDisplayName());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validateCreatedEntity(
|
||||
Glossary createdEntity, CreateGlossary createRequest, Map<String, String> authHeaders)
|
||||
throws HttpResponseException {
|
||||
validateCommonEntityFields(
|
||||
getEntityInterface(createdEntity),
|
||||
createRequest.getDescription(),
|
||||
TestUtils.getPrincipal(authHeaders),
|
||||
createRequest.getOwner());
|
||||
|
||||
// Entity specific validation
|
||||
TestUtils.validateTags(createRequest.getTags(), createdEntity.getTags());
|
||||
TestUtils.validateEntityReference(createdEntity.getFollowers());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validateUpdatedEntity(Glossary updated, CreateGlossary request, Map<String, String> authHeaders)
|
||||
throws HttpResponseException {
|
||||
validateCreatedEntity(updated, request, authHeaders);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void compareEntities(Glossary expected, Glossary patched, Map<String, String> authHeaders)
|
||||
throws HttpResponseException {
|
||||
validateCommonEntityFields(
|
||||
getEntityInterface(patched),
|
||||
expected.getDescription(),
|
||||
TestUtils.getPrincipal(authHeaders),
|
||||
expected.getOwner());
|
||||
|
||||
// Entity specific validation
|
||||
TestUtils.validateTags(expected.getTags(), patched.getTags());
|
||||
TestUtils.validateEntityReference(expected.getFollowers());
|
||||
}
|
||||
|
||||
@Override
|
||||
public GlossaryEntityInterface getEntityInterface(Glossary entity) {
|
||||
return new GlossaryEntityInterface(entity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validateGetWithDifferentFields(Glossary entity, boolean byName) throws HttpResponseException {}
|
||||
|
||||
@Override
|
||||
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
|
||||
if (expected == actual) {
|
||||
return;
|
||||
}
|
||||
assertCommonFieldChange(fieldName, expected, actual);
|
||||
}
|
||||
}
|
||||
1332
ingestion/examples/sample_data/thesauruses/hobbies-SKOS.json
Normal file
1332
ingestion/examples/sample_data/thesauruses/hobbies-SKOS.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -5,6 +5,7 @@
|
||||
"include_tables": "true",
|
||||
"include_topics": "true",
|
||||
"include_dashboards": "true",
|
||||
"include_glossary": "true",
|
||||
"limit_records": 10
|
||||
}
|
||||
},
|
||||
@ -14,6 +15,7 @@
|
||||
"index_tables": "true",
|
||||
"index_topics": "true",
|
||||
"index_dashboards": "true",
|
||||
"index_glossary": "true",
|
||||
"es_host": "localhost",
|
||||
"es_port": 9200
|
||||
}
|
||||
|
||||
@ -104,6 +104,23 @@ class TopicESDocument(BaseModel):
|
||||
doc_as_upsert: bool = True
|
||||
|
||||
|
||||
class GlossaryESDocument(BaseModel):
|
||||
"""Glossary Elastic Search Mapping doc"""
|
||||
|
||||
glossary_id: str
|
||||
glossary_name: str
|
||||
entity_type: str = "glossary"
|
||||
suggest: List[dict]
|
||||
description: Optional[str] = None
|
||||
last_updated_timestamp: Optional[int]
|
||||
tags: List[str]
|
||||
fqdn: str
|
||||
tier: Optional[str] = None
|
||||
schema_description: Optional[str] = None
|
||||
owner: str
|
||||
followers: List[str]
|
||||
|
||||
|
||||
class DashboardESDocument(BaseModel):
|
||||
"""Elastic Search Mapping doc for Dashboards"""
|
||||
|
||||
|
||||
@ -31,6 +31,7 @@ from metadata.generated.schema.entity.data.mlmodel import MlModel
|
||||
from metadata.generated.schema.entity.data.pipeline import Pipeline
|
||||
from metadata.generated.schema.entity.data.report import Report
|
||||
from metadata.generated.schema.entity.data.table import Table
|
||||
from metadata.generated.schema.entity.data.glossary import Glossary
|
||||
from metadata.generated.schema.entity.data.topic import Topic
|
||||
from metadata.generated.schema.entity.policies.policy import Policy
|
||||
from metadata.generated.schema.entity.services.dashboardService import DashboardService
|
||||
@ -162,6 +163,11 @@ class OpenMetadata(
|
||||
):
|
||||
return "/mlmodels"
|
||||
|
||||
if issubclass(
|
||||
entity, get_args(Union[Glossary, self.get_create_entity_type(Glossary)])
|
||||
):
|
||||
return "/glossary"
|
||||
|
||||
if issubclass(
|
||||
entity, get_args(Union[Chart, self.get_create_entity_type(Chart)])
|
||||
):
|
||||
|
||||
@ -26,6 +26,7 @@ from metadata.generated.schema.entity.data.dashboard import Dashboard
|
||||
from metadata.generated.schema.entity.data.database import Database
|
||||
from metadata.generated.schema.entity.data.pipeline import Pipeline, Task
|
||||
from metadata.generated.schema.entity.data.table import Column, Table
|
||||
from metadata.generated.schema.entity.data.glossary import Glossary
|
||||
from metadata.generated.schema.entity.data.topic import Topic
|
||||
from metadata.generated.schema.entity.services.dashboardService import DashboardService
|
||||
from metadata.generated.schema.entity.services.databaseService import DatabaseService
|
||||
@ -39,6 +40,7 @@ from metadata.ingestion.models.table_metadata import (
|
||||
DashboardESDocument,
|
||||
PipelineESDocument,
|
||||
TableESDocument,
|
||||
GlossaryESDocument,
|
||||
TopicESDocument,
|
||||
)
|
||||
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
||||
@ -47,6 +49,7 @@ from metadata.ingestion.sink.elasticsearch_constants import (
|
||||
DASHBOARD_ELASTICSEARCH_INDEX_MAPPING,
|
||||
PIPELINE_ELASTICSEARCH_INDEX_MAPPING,
|
||||
TABLE_ELASTICSEARCH_INDEX_MAPPING,
|
||||
GLOSSARY_ELASTICSEARCH_INDEX_MAPPING,
|
||||
TOPIC_ELASTICSEARCH_INDEX_MAPPING,
|
||||
)
|
||||
|
||||
@ -64,10 +67,12 @@ class ElasticSearchConfig(ConfigModel):
|
||||
es_password: Optional[str] = None
|
||||
index_tables: Optional[bool] = True
|
||||
index_topics: Optional[bool] = True
|
||||
index_glossary: Optional[bool] = True
|
||||
index_dashboards: Optional[bool] = True
|
||||
index_pipelines: Optional[bool] = True
|
||||
index_dbt_models: Optional[bool] = True
|
||||
table_index_name: str = "table_search_index"
|
||||
glossary_index_name: str = "glossary_search_index"
|
||||
topic_index_name: str = "topic_search_index"
|
||||
dashboard_index_name: str = "dashboard_search_index"
|
||||
pipeline_index_name: str = "pipeline_search_index"
|
||||
@ -131,6 +136,10 @@ class ElasticsearchSink(Sink[Entity]):
|
||||
self._check_or_create_index(
|
||||
self.config.table_index_name, TABLE_ELASTICSEARCH_INDEX_MAPPING
|
||||
)
|
||||
if self.config.index_glossary:
|
||||
self._check_or_create_index(
|
||||
self.config.glossary_index_name, GLOSSARY_ELASTICSEARCH_INDEX_MAPPING
|
||||
)
|
||||
if self.config.index_topics:
|
||||
self._check_or_create_index(
|
||||
self.config.topic_index_name, TOPIC_ELASTICSEARCH_INDEX_MAPPING
|
||||
@ -183,6 +192,13 @@ class ElasticsearchSink(Sink[Entity]):
|
||||
body=table_doc.json(),
|
||||
request_timeout=self.config.timeout,
|
||||
)
|
||||
if isinstance(record, Glossary):
|
||||
glossary_doc = self._create_glossary_es_doc(record)
|
||||
self.elasticsearch_client.index(
|
||||
index=self.config.glossary_index_name,
|
||||
id=str(glossary_doc.glossary_id),
|
||||
body=glossary_doc.json(),
|
||||
)
|
||||
if isinstance(record, Topic):
|
||||
topic_doc = self._create_topic_es_doc(record)
|
||||
self.elasticsearch_client.index(
|
||||
@ -443,6 +459,61 @@ class ElasticsearchSink(Sink[Entity]):
|
||||
|
||||
return pipeline_doc
|
||||
|
||||
def _create_glossary_es_doc(self, glossary: Glossary):
|
||||
fqdn = glossary.fullyQualifiedName
|
||||
suggest = [
|
||||
{
|
||||
"input": [
|
||||
glossary.displayName if glossary.displayName else glossary.name
|
||||
],
|
||||
"weight": 10,
|
||||
}
|
||||
]
|
||||
tags = set()
|
||||
timestamp = time.time()
|
||||
glossary_owner = (
|
||||
str(glossary.owner.id.__root__) if glossary.owner is not None else ""
|
||||
)
|
||||
glossary_followers = []
|
||||
if glossary.followers:
|
||||
for follower in glossary.followers.__root__:
|
||||
glossary_followers.append(str(follower.id.__root__))
|
||||
tier = None
|
||||
for glossary_tag in glossary.tags:
|
||||
if "Tier" in glossary_tag.tagFQN:
|
||||
tier = glossary_tag.tagFQN
|
||||
else:
|
||||
tags.add(glossary_tag.tagFQN)
|
||||
# tasks: List[Task] = glossary.tasks # TODO Handle Glossary words
|
||||
# task_names = []
|
||||
# task_descriptions = []
|
||||
# for task in tasks:
|
||||
# task_names.append(task.displayName)
|
||||
# if task.description is not None:
|
||||
# task_descriptions.append(task.description)
|
||||
# if tags in task and len(task.tags) > 0:
|
||||
# for col_tag in task.tags:
|
||||
# tags.add(col_tag.tagFQN)
|
||||
|
||||
glossary_doc = GlossaryESDocument(
|
||||
glossary_id=str(glossary.id.__root__),
|
||||
glossary_name=glossary.displayName
|
||||
if glossary.displayName
|
||||
else glossary.name,
|
||||
# task_names=task_names, # TODO Handle Glossary words
|
||||
# task_descriptions=task_descriptions,
|
||||
suggest=suggest,
|
||||
description=glossary.description,
|
||||
last_updated_timestamp=timestamp,
|
||||
tier=tier,
|
||||
tags=list(tags),
|
||||
fqdn=fqdn,
|
||||
owner=glossary_owner,
|
||||
followers=glossary_followers,
|
||||
)
|
||||
|
||||
return glossary_doc
|
||||
|
||||
def _get_charts(self, chart_refs: Optional[List[entityReference.EntityReference]]):
|
||||
charts = []
|
||||
if chart_refs:
|
||||
|
||||
@ -200,6 +200,48 @@ TOPIC_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
|
||||
"""
|
||||
)
|
||||
|
||||
GLOSSARY_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
|
||||
"""
|
||||
{
|
||||
"mappings":{
|
||||
"properties": {
|
||||
"glossary_name": {
|
||||
"type":"text"
|
||||
},
|
||||
"display_name": {
|
||||
"type": "text"
|
||||
},
|
||||
"owner": {
|
||||
"type": "text"
|
||||
},
|
||||
"followers": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"last_updated_timestamp": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"description": {
|
||||
"type": "text"
|
||||
},
|
||||
"tier": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"tags": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"entity_type": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"suggest": {
|
||||
"type": "completion"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
|
||||
"""
|
||||
{
|
||||
|
||||
@ -23,6 +23,7 @@ from metadata.ingestion.api.common import Entity, WorkflowContext
|
||||
from metadata.ingestion.api.source import Source, SourceStatus
|
||||
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
||||
from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
|
||||
from metadata.ingestion.ometa.openmetadata_rest import Glossary
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -33,6 +34,7 @@ class MetadataTablesRestSourceConfig(ConfigModel):
|
||||
include_tables: Optional[bool] = True
|
||||
include_topics: Optional[bool] = True
|
||||
include_dashboards: Optional[bool] = True
|
||||
include_glossary: Optional[bool] = True
|
||||
include_pipelines: Optional[bool] = True
|
||||
limit_records: int = 1000
|
||||
|
||||
@ -60,6 +62,10 @@ class MetadataSourceStatus(SourceStatus):
|
||||
self.success.append(table_name)
|
||||
logger.info("Table Scanned: %s", table_name)
|
||||
|
||||
def scanned_glossary(self, glossary_name: str) -> None:
|
||||
self.success.append(glossary_name)
|
||||
logger.info("Glossary Scanned: {}".format(glossary_name))
|
||||
|
||||
def scanned_topic(self, topic_name: str) -> None:
|
||||
"""scanned topic method
|
||||
|
||||
@ -143,6 +149,7 @@ class MetadataSource(Source[Entity]):
|
||||
def next_record(self) -> Iterable[Entity]:
|
||||
yield from self.fetch_table()
|
||||
yield from self.fetch_topic()
|
||||
yield from self.fetch_glossary()
|
||||
yield from self.fetch_dashboard()
|
||||
yield from self.fetch_pipeline()
|
||||
|
||||
@ -247,6 +254,23 @@ class MetadataSource(Source[Entity]):
|
||||
break
|
||||
after = pipeline_entities.after
|
||||
|
||||
def fetch_glossary(self) -> Glossary:
|
||||
if self.config.include_glossary:
|
||||
after = None
|
||||
while True:
|
||||
glossary_entities = self.metadata.list_entities(
|
||||
entity=Glossary,
|
||||
fields=["owner", "tags", "followers"],
|
||||
after=after,
|
||||
limit=self.config.limit_records,
|
||||
)
|
||||
for glossary in glossary_entities.entities:
|
||||
self.status.scanned_glossary(glossary.name)
|
||||
yield glossary
|
||||
if glossary_entities.after is None:
|
||||
break
|
||||
after = glossary_entities.after
|
||||
|
||||
def get_status(self) -> SourceStatus:
|
||||
return self.status
|
||||
|
||||
|
||||
@ -27,6 +27,7 @@ from metadata.generated.schema.entity.data.mlmodel import MlModel
|
||||
from metadata.generated.schema.entity.data.pipeline import Pipeline
|
||||
from metadata.generated.schema.entity.data.report import Report
|
||||
from metadata.generated.schema.entity.data.table import Table
|
||||
from metadata.generated.schema.entity.data.glossary import Glossary
|
||||
from metadata.generated.schema.entity.data.topic import Topic
|
||||
from metadata.generated.schema.entity.services.dashboardService import DashboardService
|
||||
from metadata.generated.schema.entity.services.databaseService import DatabaseService
|
||||
@ -51,6 +52,10 @@ class OMetaEndpointTest(TestCase):
|
||||
Pass Entities and test their suffix generation
|
||||
"""
|
||||
|
||||
|
||||
# Glossary
|
||||
self.assertEqual(self.metadata.get_suffix(Glossary), "/glossary")
|
||||
|
||||
# ML
|
||||
self.assertEqual(self.metadata.get_suffix(MlModel), "/mlmodels")
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user