Squash merge glossary support

This commit is contained in:
sureshms 2022-01-29 10:31:18 -08:00
parent 52ec2e3ad1
commit 17adf5debc
18 changed files with 2728 additions and 1 deletions

View File

@ -0,0 +1,13 @@
CREATE TABLE IF NOT EXISTS glossary_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
timestamp BIGINT,
PRIMARY KEY (id),
UNIQUE KEY unique_name(fullyQualifiedName),
INDEX (updatedBy),
INDEX (updatedAt)
);

View File

@ -70,6 +70,7 @@ public final class Entity {
public static final String UNUSED = "unused"; public static final String UNUSED = "unused";
public static final String BOTS = "bots"; public static final String BOTS = "bots";
public static final String LOCATION = "location"; public static final String LOCATION = "location";
public static final String GLOSSARY = "glossary";
// //
// Policies // Policies

View File

@ -12,6 +12,7 @@ import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.function.Predicate; import java.util.function.Predicate;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import javax.ws.rs.core.Response;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
@ -31,6 +32,7 @@ import org.elasticsearch.client.indices.PutMappingRequest;
import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.XContentType;
import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Dashboard; import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.entity.data.Pipeline; import org.openmetadata.catalog.entity.data.Pipeline;
import org.openmetadata.catalog.entity.data.Table; import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.entity.data.Topic; import org.openmetadata.catalog.entity.data.Topic;
@ -63,7 +65,8 @@ public class ElasticSearchIndexDefinition {
TABLE_SEARCH_INDEX("table_search_index", "/elasticsearch/table_index_mapping.json"), TABLE_SEARCH_INDEX("table_search_index", "/elasticsearch/table_index_mapping.json"),
TOPIC_SEARCH_INDEX("topic_search_index", "/elasticsearch/topic_index_mapping.json"), TOPIC_SEARCH_INDEX("topic_search_index", "/elasticsearch/topic_index_mapping.json"),
DASHBOARD_SEARCH_INDEX("dashboard_search_index", "/elasticsearch/dashboard_index_mapping.json"), DASHBOARD_SEARCH_INDEX("dashboard_search_index", "/elasticsearch/dashboard_index_mapping.json"),
PIPELINE_SEARCH_INDEX("pipeline_search_index", "/elasticsearch/pipeline_index_mapping.json"); PIPELINE_SEARCH_INDEX("pipeline_search_index", "/elasticsearch/pipeline_index_mapping.json"),
GLOSSARY_SEARCH_INDEX("glossary_search_index", "/elasticsearch/glossary_index_mapping.json");
public final String indexName; public final String indexName;
public final String indexMappingFile; public final String indexMappingFile;
@ -189,6 +192,8 @@ public class ElasticSearchIndexDefinition {
return ElasticSearchIndexType.PIPELINE_SEARCH_INDEX; return ElasticSearchIndexType.PIPELINE_SEARCH_INDEX;
} else if (type.equalsIgnoreCase(Entity.TOPIC)) { } else if (type.equalsIgnoreCase(Entity.TOPIC)) {
return ElasticSearchIndexType.TOPIC_SEARCH_INDEX; return ElasticSearchIndexType.TOPIC_SEARCH_INDEX;
} else if (type.equalsIgnoreCase(Entity.TOPIC)) {
return ElasticSearchIndexType.GLOSSARY_SEARCH_INDEX;
} }
throw new RuntimeException("Failed to find index doc for type " + type); throw new RuntimeException("Failed to find index doc for type " + type);
} }
@ -707,3 +712,68 @@ class PipelineESIndex extends ElasticSearchIndex {
return pipelineESIndexBuilder; return pipelineESIndexBuilder;
} }
} }
@EqualsAndHashCode(callSuper = true)
@Getter
@SuperBuilder(builderMethodName = "internalBuilder")
@Value
@JsonInclude(JsonInclude.Include.NON_NULL)
class GlossaryESIndex extends ElasticSearchIndex {
@JsonProperty("glossary_id")
String glossaryId;
public static GlossaryESIndexBuilder builder(Glossary glossary, int responseCode) {
List<String> tags = new ArrayList<>();
List<String> taskNames = new ArrayList<>();
List<String> taskDescriptions = new ArrayList<>();
List<ElasticSearchSuggest> suggest = new ArrayList<>();
suggest.add(ElasticSearchSuggest.builder().input(glossary.getFullyQualifiedName()).weight(5).build());
suggest.add(ElasticSearchSuggest.builder().input(glossary.getDisplayName()).weight(10).build());
if (glossary.getTags() != null) {
glossary.getTags().forEach(tag -> tags.add(tag.getTagFQN()));
}
Long updatedTimestamp = glossary.getUpdatedAt();
ParseTags parseTags = new ParseTags(tags);
String description = glossary.getDescription() != null ? glossary.getDescription() : "";
String displayName = glossary.getDisplayName() != null ? glossary.getDisplayName() : "";
GlossaryESIndexBuilder builder =
internalBuilder()
.glossaryId(glossary.getId().toString())
.name(glossary.getDisplayName())
.displayName(description)
.description(displayName)
.fqdn(glossary.getFullyQualifiedName())
.lastUpdatedTimestamp(updatedTimestamp)
.entityType("glossary")
.suggest(suggest)
.tags(parseTags.tags)
.tier(parseTags.tierTag);
if (glossary.getFollowers() != null) {
builder.followers(
glossary.getFollowers().stream().map(item -> item.getId().toString()).collect(Collectors.toList()));
} else if (responseCode == Response.Status.CREATED.getStatusCode()) {
builder.followers(Collections.emptyList());
}
if (glossary.getOwner() != null) {
builder.owner(glossary.getOwner().getId().toString());
}
ESChangeDescription esChangeDescription = null;
if (glossary.getChangeDescription() != null) {
esChangeDescription =
ESChangeDescription.builder().updatedAt(updatedTimestamp).updatedBy(glossary.getUpdatedBy()).build();
esChangeDescription.setFieldsAdded(glossary.getChangeDescription().getFieldsAdded());
esChangeDescription.setFieldsDeleted(glossary.getChangeDescription().getFieldsDeleted());
esChangeDescription.setFieldsUpdated(glossary.getChangeDescription().getFieldsUpdated());
} else if (responseCode == Response.Status.CREATED.getStatusCode()) {
esChangeDescription =
ESChangeDescription.builder().updatedAt(updatedTimestamp).updatedBy(glossary.getUpdatedBy()).build();
}
builder.changeDescriptions(esChangeDescription != null ? List.of(esChangeDescription) : null);
return builder;
}
}

View File

@ -33,6 +33,7 @@ import org.openmetadata.catalog.entity.Bots;
import org.openmetadata.catalog.entity.data.Chart; import org.openmetadata.catalog.entity.data.Chart;
import org.openmetadata.catalog.entity.data.Dashboard; import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.entity.data.Database; import org.openmetadata.catalog.entity.data.Database;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.entity.data.Location; import org.openmetadata.catalog.entity.data.Location;
import org.openmetadata.catalog.entity.data.Metrics; import org.openmetadata.catalog.entity.data.Metrics;
import org.openmetadata.catalog.entity.data.MlModel; import org.openmetadata.catalog.entity.data.MlModel;
@ -58,6 +59,7 @@ import org.openmetadata.catalog.jdbi3.DashboardRepository.DashboardEntityInterfa
import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface; import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface; import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface;
import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface; import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface;
import org.openmetadata.catalog.jdbi3.LocationRepository.LocationEntityInterface; import org.openmetadata.catalog.jdbi3.LocationRepository.LocationEntityInterface;
import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface; import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface;
import org.openmetadata.catalog.jdbi3.MetricsRepository.MetricsEntityInterface; import org.openmetadata.catalog.jdbi3.MetricsRepository.MetricsEntityInterface;
@ -134,6 +136,9 @@ public interface CollectionDAO {
@CreateSqlObject @CreateSqlObject
MlModelDAO mlModelDAO(); MlModelDAO mlModelDAO();
@CreateSqlObject
GlossaryDAO glossaryDAO();
@CreateSqlObject @CreateSqlObject
BotsDAO botsDAO(); BotsDAO botsDAO();
@ -689,6 +694,28 @@ public interface CollectionDAO {
} }
} }
interface GlossaryDAO extends EntityDAO<Glossary> {
@Override
default String getTableName() {
return "glossary_entity";
}
@Override
default Class<Glossary> getEntityClass() {
return Glossary.class;
}
@Override
default String getNameColumn() {
return "fullyQualifiedName";
}
@Override
default EntityReference getEntityReference(Glossary entity) {
return new GlossaryEntityInterface(entity).getEntityReference();
}
}
interface AirflowPipelineDAO extends EntityDAO<AirflowPipeline> { interface AirflowPipelineDAO extends EntityDAO<AirflowPipeline> {
@Override @Override
default String getTableName() { default String getTableName() {

View File

@ -0,0 +1,271 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import java.io.IOException;
import java.net.URI;
import java.text.ParseException;
import java.util.List;
import java.util.UUID;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.resources.glossary.GlossaryResource;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.JsonUtils;
public class GlossaryRepository extends EntityRepository<Glossary> {
private static final Fields GLOSSARY_UPDATE_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "owner,tags");
private static final Fields GLOSSARY_PATCH_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "owner,tags");
private final CollectionDAO dao;
public GlossaryRepository(CollectionDAO dao) {
super(
GlossaryResource.COLLECTION_PATH,
Entity.GLOSSARY,
Glossary.class,
dao.glossaryDAO(),
dao,
GLOSSARY_PATCH_FIELDS,
GLOSSARY_UPDATE_FIELDS,
true,
true,
false);
this.dao = dao;
}
public static String getFQN(Glossary glossary) {
return (glossary.getName());
}
@Transaction
public EntityReference getOwnerReference(Glossary glossary) throws IOException {
return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), glossary.getOwner());
}
@Override
public Glossary setFields(Glossary glossary, Fields fields) throws IOException, ParseException {
glossary.setDisplayName(glossary.getDisplayName());
glossary.setOwner(fields.contains("owner") ? getOwner(glossary) : null);
glossary.setFollowers(fields.contains("followers") ? getFollowers(glossary) : null);
glossary.setTags(fields.contains("tags") ? getTags(glossary.getFullyQualifiedName()) : null);
glossary.setUsageSummary(
fields.contains("usageSummary") ? EntityUtil.getLatestUsage(dao.usageDAO(), glossary.getId()) : null);
return glossary;
}
@Override
public void prepare(Glossary glossary) throws IOException {
glossary.setFullyQualifiedName(getFQN(glossary));
EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), glossary.getOwner()); // Validate owner
glossary.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), glossary.getTags()));
}
@Override
public void storeEntity(Glossary glossary, boolean update) throws IOException {
// Relationships and fields such as href are derived and not stored as part of json
EntityReference owner = glossary.getOwner();
List<TagLabel> tags = glossary.getTags();
// Don't store owner, dashboard, href and tags as JSON. Build it on the fly based on relationships
glossary.withOwner(null).withHref(null).withTags(null);
if (update) {
dao.glossaryDAO().update(glossary.getId(), JsonUtils.pojoToJson(glossary));
} else {
dao.glossaryDAO().insert(glossary);
}
// Restore the relationships
glossary.withOwner(owner).withTags(tags);
}
@Override
public void restorePatchAttributes(Glossary original, Glossary updated) {}
@Override
public EntityInterface<Glossary> getEntityInterface(Glossary entity) {
return new GlossaryEntityInterface(entity);
}
@Override
public void storeRelationships(Glossary glossary) {
setOwner(glossary, glossary.getOwner());
applyTags(glossary);
}
@Override
public EntityUpdater getUpdater(Glossary original, Glossary updated, Operation operation) {
return new GlossaryUpdater(original, updated, operation);
}
public static class GlossaryEntityInterface implements EntityInterface<Glossary> {
private final Glossary entity;
public GlossaryEntityInterface(Glossary entity) {
this.entity = entity;
}
@Override
public UUID getId() {
return entity.getId();
}
@Override
public String getDescription() {
return entity.getDescription();
}
@Override
public String getDisplayName() {
return entity.getDisplayName();
}
@Override
public Boolean isDeleted() {
return entity.getDeleted();
}
@Override
public EntityReference getOwner() {
return entity.getOwner();
}
@Override
public String getFullyQualifiedName() {
return entity.getFullyQualifiedName();
}
@Override
public List<TagLabel> getTags() {
return entity.getTags();
}
public String getSkos() {
return entity.getSkos();
}
@Override
public Double getVersion() {
return entity.getVersion();
}
@Override
public String getUpdatedBy() {
return entity.getUpdatedBy();
}
@Override
public long getUpdatedAt() {
return entity.getUpdatedAt();
}
@Override
public URI getHref() {
return entity.getHref();
}
@Override
public List<EntityReference> getFollowers() {
return entity.getFollowers();
}
@Override
public ChangeDescription getChangeDescription() {
return entity.getChangeDescription();
}
@Override
public EntityReference getEntityReference() {
return new EntityReference()
.withId(getId())
.withName(getFullyQualifiedName())
.withDescription(getDescription())
.withDisplayName(getDisplayName())
.withType(Entity.GLOSSARY);
}
@Override
public Glossary getEntity() {
return entity;
}
@Override
public EntityReference getContainer() {
return null;
}
@Override
public void setId(UUID id) {
entity.setId(id);
}
@Override
public void setDescription(String description) {
entity.setDescription(description);
}
@Override
public void setDisplayName(String displayName) {
entity.setDisplayName(displayName);
}
@Override
public void setUpdateDetails(String updatedBy, long updatedAt) {
entity.setUpdatedBy(updatedBy);
entity.setUpdatedAt(updatedAt);
}
@Override
public void setChangeDescription(Double newVersion, ChangeDescription changeDescription) {
entity.setVersion(newVersion);
entity.setChangeDescription(changeDescription);
}
@Override
public void setOwner(EntityReference owner) {
entity.setOwner(owner);
}
@Override
public void setDeleted(boolean flag) {
entity.setDeleted(flag);
}
@Override
public Glossary withHref(URI href) {
return entity.withHref(href);
}
@Override
public void setTags(List<TagLabel> tags) {
entity.setTags(tags);
}
}
/** Handles entity updated from PUT and POST operation. */
public class GlossaryUpdater extends EntityUpdater {
public GlossaryUpdater(Glossary original, Glossary updated, Operation operation) {
super(original, updated, operation);
}
}
}

View File

@ -0,0 +1,436 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.resources.glossary;
import com.google.inject.Inject;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.PATCH;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateGlossary;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.jdbi3.CollectionDAO;
import org.openmetadata.catalog.jdbi3.GlossaryRepository;
import org.openmetadata.catalog.resources.Collection;
import org.openmetadata.catalog.security.Authorizer;
import org.openmetadata.catalog.security.SecurityUtil;
import org.openmetadata.catalog.type.EntityHistory;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.Include;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.RestUtil;
import org.openmetadata.catalog.util.RestUtil.DeleteResponse;
import org.openmetadata.catalog.util.RestUtil.PatchResponse;
import org.openmetadata.catalog.util.RestUtil.PutResponse;
import org.openmetadata.catalog.util.ResultList;
@Path("/v1/glossary")
@Api(value = "Glossary collection", tags = "Glossary collection")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "glossary")
public class GlossaryResource {
public static final String COLLECTION_PATH = "v1/glossary/";
private final GlossaryRepository dao;
private final Authorizer authorizer;
public static void addHref(UriInfo uriInfo, EntityReference ref) {
ref.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, ref.getId()));
}
public static List<Glossary> addHref(UriInfo uriInfo, List<Glossary> glossary) {
Optional.ofNullable(glossary).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i));
return glossary;
}
public static Glossary addHref(UriInfo uriInfo, Glossary glossary) {
glossary.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, glossary.getId()));
Entity.withHref(uriInfo, glossary.getOwner());
Entity.withHref(uriInfo, glossary.getFollowers());
return glossary;
}
@Inject
public GlossaryResource(CollectionDAO dao, Authorizer authorizer) {
Objects.requireNonNull(dao, "GlossaryRepository must not be null");
this.dao = new GlossaryRepository(dao);
this.authorizer = authorizer;
}
public static class GlossaryList extends ResultList<Glossary> {
@SuppressWarnings("unused")
GlossaryList() {
// Empty constructor needed for deserialization
}
public GlossaryList(List<Glossary> data, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(data, beforeCursor, afterCursor, total);
}
}
static final String FIELDS = "owner,dashboard,definition,followers,tags,usageSummary,skos";
public static final List<String> FIELD_LIST = Arrays.asList(FIELDS.replaceAll(" ", "").split(","));
@GET
@Valid
@Operation(
summary = "List Glossary",
tags = "glossary",
description =
"Get a list of glossary. Use `fields` parameter to get only necessary fields. "
+ " Use cursor-based pagination to limit the number "
+ "entries in the list using `limit` and `before` or `after` query params.",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of glossary",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryList.class)))
})
public ResultList<Glossary> list(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(description = "Limit the number glossary returned. (1 to 1000000, " + "default = 10)")
@DefaultValue("10")
@Min(1)
@Max(1000000)
@QueryParam("limit")
int limitParam,
@Parameter(description = "Returns list of glossary before this cursor", schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(description = "Returns list of glossary after this cursor", schema = @Schema(type = "string"))
@QueryParam("after")
String after,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException, GeneralSecurityException, ParseException {
RestUtil.validateCursors(before, after);
Fields fields = new Fields(FIELD_LIST, fieldsParam);
ResultList<Glossary> glossary;
if (before != null) { // Reverse paging
glossary = dao.listBefore(uriInfo, fields, null, limitParam, before, include); // Ask for one extra entry
} else { // Forward paging or first page
glossary = dao.listAfter(uriInfo, fields, null, limitParam, after, include);
}
addHref(uriInfo, glossary.getData());
return glossary;
}
@GET
@Path("/{id}")
@Operation(
summary = "Get a glossary",
tags = "glossary",
description = "Get a glossary by `id`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The glossary",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
@ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found")
})
public Glossary get(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") String id,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException, ParseException {
Fields fields = new Fields(FIELD_LIST, fieldsParam);
return addHref(uriInfo, dao.get(uriInfo, id, fields, include));
}
@GET
@Path("/name/{fqn}")
@Operation(
summary = "Get a glossary by name",
tags = "glossary",
description = "Get a glossary by fully qualified name.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The glossary",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
@ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found")
})
public Glossary getByName(
@Context UriInfo uriInfo,
@PathParam("fqn") String fqn,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException, ParseException {
Fields fields = new Fields(FIELD_LIST, fieldsParam);
Glossary glossary = dao.getByName(uriInfo, fqn, fields, include);
return addHref(uriInfo, glossary);
}
@GET
@Path("/{id}/versions")
@Operation(
summary = "List glossary versions",
tags = "glossary",
description = "Get a list of all the versions of a glossary identified by `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of glossary versions",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class)))
})
public EntityHistory listVersions(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id)
throws IOException, ParseException {
return dao.listVersions(id);
}
@GET
@Path("/{id}/versions/{version}")
@Operation(
summary = "Get a version of the glossary",
tags = "glossary",
description = "Get a version of the glossary by given `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "glossary",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
@ApiResponse(
responseCode = "404",
description = "Glossary for instance {id} and version {version} is " + "not found")
})
public Glossary getVersion(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id,
@Parameter(
description = "glossary version number in the form `major`.`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version")
String version)
throws IOException, ParseException {
return dao.getVersion(id, version);
}
@POST
@Operation(
summary = "Create a glossary",
tags = "glossary",
description = "Create a new glossary.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The glossary",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = CreateGlossary.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response create(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create)
throws IOException, ParseException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
Glossary glossary = getGlossary(securityContext, create);
glossary = addHref(uriInfo, dao.create(uriInfo, glossary));
return Response.created(glossary.getHref()).entity(glossary).build();
}
@PATCH
@Path("/{id}")
@Operation(
summary = "Update a glossary",
tags = "glossary",
description = "Update an existing glossary using JsonPatch.",
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
public Response updateDescription(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") String id,
@RequestBody(
description = "JsonPatch with array of operations",
content =
@Content(
mediaType = MediaType.APPLICATION_JSON_PATCH_JSON,
examples = {
@ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]")
}))
JsonPatch patch)
throws IOException, ParseException {
Fields fields = new Fields(FIELD_LIST, FIELDS);
Glossary glossary = dao.get(uriInfo, id, fields);
SecurityUtil.checkAdminRoleOrPermissions(
authorizer, securityContext, dao.getEntityInterface(glossary).getEntityReference(), patch);
PatchResponse<Glossary> response =
dao.patch(uriInfo, UUID.fromString(id), securityContext.getUserPrincipal().getName(), patch);
addHref(uriInfo, response.getEntity());
return response.toResponse();
}
@PUT
@Operation(
summary = "Create or update a glossary",
tags = "glossary",
description = "Create a new glossary, if it does not exist or update an existing glossary.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The glossary",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response createOrUpdate(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create)
throws IOException, ParseException {
Glossary glossary = getGlossary(securityContext, create);
PutResponse<Glossary> response = dao.createOrUpdate(uriInfo, glossary);
addHref(uriInfo, response.getEntity());
return response.toResponse();
}
@PUT
@Path("/{id}/followers")
@Operation(
summary = "Add a follower",
tags = "glossary",
description = "Add a user identified by `userId` as follower of this glossary",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "glossary for instance {id} is not found")
})
public Response addFollower(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the glossary", schema = @Schema(type = "string")) @PathParam("id") String id,
@Parameter(description = "Id of the user to be added as follower", schema = @Schema(type = "string"))
String userId)
throws IOException, ParseException {
return dao.addFollower(securityContext.getUserPrincipal().getName(), UUID.fromString(id), UUID.fromString(userId))
.toResponse();
}
@DELETE
@Path("/{id}/followers/{userId}")
@Operation(
summary = "Remove a follower",
tags = "glossary",
description = "Remove the user identified `userId` as a follower of the glossary.")
public Response deleteFollower(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the glossary", schema = @Schema(type = "string")) @PathParam("id") String id,
@Parameter(description = "Id of the user being removed as follower", schema = @Schema(type = "string"))
@PathParam("userId")
String userId)
throws IOException, ParseException {
return dao.deleteFollower(
securityContext.getUserPrincipal().getName(), UUID.fromString(id), UUID.fromString(userId))
.toResponse();
}
@DELETE
@Path("/{id}")
@Operation(
summary = "Delete a Glossary",
tags = "glossary",
description = "Delete a glossary by `id`.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "glossary for instance {id} is not found")
})
public Response delete(@Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("id") String id)
throws IOException, ParseException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
DeleteResponse<Glossary> response = dao.delete(securityContext.getUserPrincipal().getName(), id);
return response.toResponse();
}
private Glossary getGlossary(SecurityContext securityContext, CreateGlossary create) {
return new Glossary()
.withId(UUID.randomUUID())
.withName(create.getName())
.withDisplayName(create.getDisplayName())
.withDescription(create.getDescription())
.withSkos(create.getSkos())
.withTags(create.getTags())
.withOwner(create.getOwner())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(System.currentTimeMillis());
}
}

View File

@ -146,6 +146,9 @@ public class SearchResource {
case "table_search_index": case "table_search_index":
searchSourceBuilder = buildTableSearchBuilder(query, from, size); searchSourceBuilder = buildTableSearchBuilder(query, from, size);
break; break;
case "glossary_search_index":
searchSourceBuilder = buildGlossarySearchBuilder(query, from, size);
break;
default: default:
searchSourceBuilder = buildAggregateSearchBuilder(query, from, size); searchSourceBuilder = buildAggregateSearchBuilder(query, from, size);
break; break;
@ -353,4 +356,27 @@ public class SearchResource {
return searchSourceBuilder; return searchSourceBuilder;
} }
private SearchSourceBuilder buildGlossarySearchBuilder(String query, int from, int size) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
HighlightBuilder.Field highlightGlossaryName = new HighlightBuilder.Field("glossary_name");
highlightGlossaryName.highlighterType("unified");
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field("description");
highlightDescription.highlighterType("unified");
HighlightBuilder hb = new HighlightBuilder();
hb.field(highlightDescription);
hb.field(highlightGlossaryName);
hb.preTags("<span class=\"text-highlighter\">");
hb.postTags("</span>");
searchSourceBuilder
.query(QueryBuilders.queryStringQuery(query).field("glossary_name", 5.0f).field("description").lenient(true))
.aggregation(AggregationBuilders.terms("EntityType").field("entity_type"))
.aggregation(AggregationBuilders.terms("Tier").field("tier"))
.aggregation(AggregationBuilders.terms("Tags").field("tags"))
.highlighter(hb)
.from(from)
.size(size);
return searchSourceBuilder;
}
} }

View File

@ -0,0 +1,40 @@
{
"$id": "https://open-metadata.org/schema/api/data/createGlossary.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Create Glossary entity request",
"description": "Create Glossary entity request",
"type": "object",
"properties" : {
"name": {
"description": "Name that identifies this glossary.",
"type": "string",
"minLength": 1,
"maxLength": 128
},
"displayName": {
"description": "Display Name that identifies this glossary.",
"type": "string"
},
"description": {
"description": "Description of the glossary instance.",
"type": "string"
},
"skos": {
"description": "SKOS data in JSON-LD format",
"type": "string"
},
"tags": {
"description": "Tags for this glossary",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"owner": {
"description": "Owner of this glossary",
"$ref": "../../type/entityReference.json"
}
},
"required": ["name"]
}

View File

@ -0,0 +1,85 @@
{
"$id": "https://open-metadata.org/schema/entity/data/glossary.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Glossary",
"description": "This schema defines the Glossary entity based on SKOS.",
"type": "object",
"properties" : {
"id": {
"description": "Unique identifier of a glossary instance.",
"$ref": "../../type/basic.json#/definitions/uuid"
},
"name": {
"description": "Name that identifies this glossary.",
"type": "string",
"minLength": 1,
"maxLength": 128
},
"fullyQualifiedName": {
"description": "A unique name that identifies a glossary.",
"type": "string",
"minLength": 1,
"maxLength": 256
},
"displayName": {
"description": "Display Name that identifies this glossary.",
"type": "string"
},
"version" : {
"description": "Metadata version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/entityVersion"
},
"updatedAt" : {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"updatedBy" : {
"description": "User who made the update.",
"type": "string"
},
"description": {
"description": "Description of the glossary.",
"type": "string"
},
"skos": {
"description": "SKOS data in JSON-LD format",
"type": "string"
},
"href": {
"description": "Link to the resource corresponding to this entity.",
"$ref": "../../type/basic.json#/definitions/href"
},
"owner": {
"description": "Owner of this glossary.",
"$ref": "../../type/entityReference.json"
},
"followers": {
"description": "Followers of this glossary.",
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList"
},
"tags": {
"description": "Tags for this glossary.",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"usageSummary" : {
"description": "Latest usage information for this glossary.",
"$ref": "../../type/usageDetails.json",
"default": null
},
"changeDescription": {
"description" : "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted" : {
"description": "When `true` indicates the entity has been soft deleted.",
"type" : "boolean",
"default": false
}
},
"required": ["id", "name"]
}

View File

@ -0,0 +1,259 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.resources.glossary;
import static javax.ws.rs.core.Response.Status.FORBIDDEN;
import static javax.ws.rs.core.Response.Status.NOT_FOUND;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.openmetadata.catalog.security.SecurityUtil.authHeaders;
import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.catalog.util.TestUtils.NON_EXISTENT_ENTITY;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Map;
import java.util.UUID;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.TestMethodOrder;
import org.openmetadata.catalog.CatalogApplicationTest;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateGlossary;
import org.openmetadata.catalog.entity.data.Glossary;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface;
import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.TestUtils;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlossary> {
public GlossaryResourceTest() {
super(
Entity.GLOSSARY,
Glossary.class,
GlossaryResource.GlossaryList.class,
"glossary",
GlossaryResource.FIELDS,
false,
true,
true,
true);
}
@BeforeAll
public void setup(TestInfo test) throws IOException, URISyntaxException {
super.setup(test);
}
@Override
public CreateGlossary createRequest(String name, String description, String displayName, EntityReference owner) {
return new CreateGlossary()
.withName(name)
.withDescription(description)
.withDisplayName(displayName)
.withOwner(owner);
}
@Override
public EntityReference getContainer(CreateGlossary createRequest) {
return null;
}
// TODO: Entity tests
// @Test
// public void patch_entityAttributes_200_ok()
// {}
// @Test
// public void put_entityCreate_200()
// {}
// @Test
// public void put_entityCreate_as_owner_200()
// {}
// @Test
// public void put_entityEmptyDescriptionUpdate_200()
// {}
// @Test
// public void put_entityNullDescriptionUpdate_200()
// {}
// @Test
// public void put_entityUpdateOwner_200()
// {}
// @Test
// public void put_entityUpdateWithNoChange_200()
// {}
@Test
public void post_validGlossary_200_OK(TestInfo test) throws IOException {
CreateGlossary create = create(test).withDescription("description");
createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
}
@Test
public void post_glossaryWithUserOwner_200_ok(TestInfo test) throws IOException {
createAndCheckEntity(create(test).withOwner(USER_OWNER1), ADMIN_AUTH_HEADERS);
}
@Test
public void post_glossaryWithTeamOwner_200_ok(TestInfo test) throws IOException {
createAndCheckEntity(create(test).withOwner(TEAM_OWNER1), ADMIN_AUTH_HEADERS);
}
// @Test
// public void post_glossaryWithInvalidOwnerType_4xx(TestInfo test) {
// EntityReference owner = new EntityReference().withId(TEAM1.getId()); /* No owner type is set */
// CreateGlossary create = create(test).withOwner(owner);
// HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
// createEntity(create, ADMIN_AUTH_HEADERS));
// TestUtils.assertResponseContains(exception, BAD_REQUEST, "type must not be null");
// }
@Test
public void post_glossary_as_non_admin_401(TestInfo test) {
CreateGlossary create = create(test);
HttpResponseException exception =
assertThrows(HttpResponseException.class, () -> createEntity(create, authHeaders("test@open-metadata.org")));
assertResponse(exception, FORBIDDEN, "Principal: CatalogPrincipal{name='test'} is not admin");
}
@Test
public void get_nonExistentGlossary_404_notFound() {
HttpResponseException exception =
assertThrows(HttpResponseException.class, () -> getEntity(NON_EXISTENT_ENTITY, ADMIN_AUTH_HEADERS));
assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, NON_EXISTENT_ENTITY));
}
@Test
public void delete_glossary_200_ok(TestInfo test) throws HttpResponseException {
Glossary glossary = createEntity(create(test), ADMIN_AUTH_HEADERS);
deleteGlossary(glossary.getId(), ADMIN_AUTH_HEADERS);
}
// @Test
// public void delete_glossary_as_non_admin_401(TestInfo test) throws HttpResponseException {
// Glossary glossary = createEntity(create(test), ADMIN_AUTH_HEADERS);
// HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
// deleteGlossary(glossary.getId(), authHeaders("test@open-metadata.org")));
// assertResponse(exception, FORBIDDEN, "Principal: CatalogPrincipal{name='test'} is not admin");
// }
@Test
public void delete_nonExistentGlossary_404() {
HttpResponseException exception =
assertThrows(HttpResponseException.class, () -> getEntity(NON_EXISTENT_ENTITY, ADMIN_AUTH_HEADERS));
assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, NON_EXISTENT_ENTITY));
}
// @Test
// public void put_GlossaryCreate_200(TestInfo test) throws HttpResponseException {
// // Create a new Glossary with PUT
// CreateGlossary request = create(test).withOwner(USER_OWNER1);
// updateAndCheckGlossary(null, request.withName(test.getDisplayName()).withDescription(null), CREATED,
// ADMIN_AUTH_HEADERS, NO_CHANGE);
// }
public static CreateGlossary create(TestInfo test) {
return create(test, 0);
}
public static CreateGlossary create(TestInfo test, int index) {
return new CreateGlossary().withName(getGlossaryName(test, index));
}
/**
* A method variant to be called form other tests to create a glossary without depending on Database, DatabaseService
* set up in the {@code setup()} method
*/
public Glossary createEntity(TestInfo test, int index) throws IOException {
CreateGlossary create = new CreateGlossary().withName(getGlossaryName(test, index));
return createEntity(create, ADMIN_AUTH_HEADERS);
}
private void deleteGlossary(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
TestUtils.delete(CatalogApplicationTest.getResource("glossary/" + id), authHeaders);
// Check to make sure database entity does not exist
HttpResponseException exception = assertThrows(HttpResponseException.class, () -> getEntity(id, authHeaders));
assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, id));
}
public static String getGlossaryName(TestInfo test, int index) {
return String.format("glossary%d_%s", index, test.getDisplayName());
}
@Override
public void validateCreatedEntity(
Glossary createdEntity, CreateGlossary createRequest, Map<String, String> authHeaders)
throws HttpResponseException {
validateCommonEntityFields(
getEntityInterface(createdEntity),
createRequest.getDescription(),
TestUtils.getPrincipal(authHeaders),
createRequest.getOwner());
// Entity specific validation
TestUtils.validateTags(createRequest.getTags(), createdEntity.getTags());
TestUtils.validateEntityReference(createdEntity.getFollowers());
}
@Override
public void validateUpdatedEntity(Glossary updated, CreateGlossary request, Map<String, String> authHeaders)
throws HttpResponseException {
validateCreatedEntity(updated, request, authHeaders);
}
@Override
public void compareEntities(Glossary expected, Glossary patched, Map<String, String> authHeaders)
throws HttpResponseException {
validateCommonEntityFields(
getEntityInterface(patched),
expected.getDescription(),
TestUtils.getPrincipal(authHeaders),
expected.getOwner());
// Entity specific validation
TestUtils.validateTags(expected.getTags(), patched.getTags());
TestUtils.validateEntityReference(expected.getFollowers());
}
@Override
public GlossaryEntityInterface getEntityInterface(Glossary entity) {
return new GlossaryEntityInterface(entity);
}
@Override
public void validateGetWithDifferentFields(Glossary entity, boolean byName) throws HttpResponseException {}
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
if (expected == actual) {
return;
}
assertCommonFieldChange(fieldName, expected, actual);
}
}

File diff suppressed because it is too large Load Diff

View File

@ -5,6 +5,7 @@
"include_tables": "true", "include_tables": "true",
"include_topics": "true", "include_topics": "true",
"include_dashboards": "true", "include_dashboards": "true",
"include_glossary": "true",
"limit_records": 10 "limit_records": 10
} }
}, },
@ -14,6 +15,7 @@
"index_tables": "true", "index_tables": "true",
"index_topics": "true", "index_topics": "true",
"index_dashboards": "true", "index_dashboards": "true",
"index_glossary": "true",
"es_host": "localhost", "es_host": "localhost",
"es_port": 9200 "es_port": 9200
} }

View File

@ -104,6 +104,23 @@ class TopicESDocument(BaseModel):
doc_as_upsert: bool = True doc_as_upsert: bool = True
class GlossaryESDocument(BaseModel):
"""Glossary Elastic Search Mapping doc"""
glossary_id: str
glossary_name: str
entity_type: str = "glossary"
suggest: List[dict]
description: Optional[str] = None
last_updated_timestamp: Optional[int]
tags: List[str]
fqdn: str
tier: Optional[str] = None
schema_description: Optional[str] = None
owner: str
followers: List[str]
class DashboardESDocument(BaseModel): class DashboardESDocument(BaseModel):
"""Elastic Search Mapping doc for Dashboards""" """Elastic Search Mapping doc for Dashboards"""

View File

@ -31,6 +31,7 @@ from metadata.generated.schema.entity.data.mlmodel import MlModel
from metadata.generated.schema.entity.data.pipeline import Pipeline from metadata.generated.schema.entity.data.pipeline import Pipeline
from metadata.generated.schema.entity.data.report import Report from metadata.generated.schema.entity.data.report import Report
from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.data.table import Table
from metadata.generated.schema.entity.data.glossary import Glossary
from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.data.topic import Topic
from metadata.generated.schema.entity.policies.policy import Policy from metadata.generated.schema.entity.policies.policy import Policy
from metadata.generated.schema.entity.services.dashboardService import DashboardService from metadata.generated.schema.entity.services.dashboardService import DashboardService
@ -162,6 +163,11 @@ class OpenMetadata(
): ):
return "/mlmodels" return "/mlmodels"
if issubclass(
entity, get_args(Union[Glossary, self.get_create_entity_type(Glossary)])
):
return "/glossary"
if issubclass( if issubclass(
entity, get_args(Union[Chart, self.get_create_entity_type(Chart)]) entity, get_args(Union[Chart, self.get_create_entity_type(Chart)])
): ):

View File

@ -26,6 +26,7 @@ from metadata.generated.schema.entity.data.dashboard import Dashboard
from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.pipeline import Pipeline, Task from metadata.generated.schema.entity.data.pipeline import Pipeline, Task
from metadata.generated.schema.entity.data.table import Column, Table from metadata.generated.schema.entity.data.table import Column, Table
from metadata.generated.schema.entity.data.glossary import Glossary
from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.data.topic import Topic
from metadata.generated.schema.entity.services.dashboardService import DashboardService from metadata.generated.schema.entity.services.dashboardService import DashboardService
from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.entity.services.databaseService import DatabaseService
@ -39,6 +40,7 @@ from metadata.ingestion.models.table_metadata import (
DashboardESDocument, DashboardESDocument,
PipelineESDocument, PipelineESDocument,
TableESDocument, TableESDocument,
GlossaryESDocument,
TopicESDocument, TopicESDocument,
) )
from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.ometa.ometa_api import OpenMetadata
@ -47,6 +49,7 @@ from metadata.ingestion.sink.elasticsearch_constants import (
DASHBOARD_ELASTICSEARCH_INDEX_MAPPING, DASHBOARD_ELASTICSEARCH_INDEX_MAPPING,
PIPELINE_ELASTICSEARCH_INDEX_MAPPING, PIPELINE_ELASTICSEARCH_INDEX_MAPPING,
TABLE_ELASTICSEARCH_INDEX_MAPPING, TABLE_ELASTICSEARCH_INDEX_MAPPING,
GLOSSARY_ELASTICSEARCH_INDEX_MAPPING,
TOPIC_ELASTICSEARCH_INDEX_MAPPING, TOPIC_ELASTICSEARCH_INDEX_MAPPING,
) )
@ -64,10 +67,12 @@ class ElasticSearchConfig(ConfigModel):
es_password: Optional[str] = None es_password: Optional[str] = None
index_tables: Optional[bool] = True index_tables: Optional[bool] = True
index_topics: Optional[bool] = True index_topics: Optional[bool] = True
index_glossary: Optional[bool] = True
index_dashboards: Optional[bool] = True index_dashboards: Optional[bool] = True
index_pipelines: Optional[bool] = True index_pipelines: Optional[bool] = True
index_dbt_models: Optional[bool] = True index_dbt_models: Optional[bool] = True
table_index_name: str = "table_search_index" table_index_name: str = "table_search_index"
glossary_index_name: str = "glossary_search_index"
topic_index_name: str = "topic_search_index" topic_index_name: str = "topic_search_index"
dashboard_index_name: str = "dashboard_search_index" dashboard_index_name: str = "dashboard_search_index"
pipeline_index_name: str = "pipeline_search_index" pipeline_index_name: str = "pipeline_search_index"
@ -131,6 +136,10 @@ class ElasticsearchSink(Sink[Entity]):
self._check_or_create_index( self._check_or_create_index(
self.config.table_index_name, TABLE_ELASTICSEARCH_INDEX_MAPPING self.config.table_index_name, TABLE_ELASTICSEARCH_INDEX_MAPPING
) )
if self.config.index_glossary:
self._check_or_create_index(
self.config.glossary_index_name, GLOSSARY_ELASTICSEARCH_INDEX_MAPPING
)
if self.config.index_topics: if self.config.index_topics:
self._check_or_create_index( self._check_or_create_index(
self.config.topic_index_name, TOPIC_ELASTICSEARCH_INDEX_MAPPING self.config.topic_index_name, TOPIC_ELASTICSEARCH_INDEX_MAPPING
@ -183,6 +192,13 @@ class ElasticsearchSink(Sink[Entity]):
body=table_doc.json(), body=table_doc.json(),
request_timeout=self.config.timeout, request_timeout=self.config.timeout,
) )
if isinstance(record, Glossary):
glossary_doc = self._create_glossary_es_doc(record)
self.elasticsearch_client.index(
index=self.config.glossary_index_name,
id=str(glossary_doc.glossary_id),
body=glossary_doc.json(),
)
if isinstance(record, Topic): if isinstance(record, Topic):
topic_doc = self._create_topic_es_doc(record) topic_doc = self._create_topic_es_doc(record)
self.elasticsearch_client.index( self.elasticsearch_client.index(
@ -443,6 +459,61 @@ class ElasticsearchSink(Sink[Entity]):
return pipeline_doc return pipeline_doc
def _create_glossary_es_doc(self, glossary: Glossary):
fqdn = glossary.fullyQualifiedName
suggest = [
{
"input": [
glossary.displayName if glossary.displayName else glossary.name
],
"weight": 10,
}
]
tags = set()
timestamp = time.time()
glossary_owner = (
str(glossary.owner.id.__root__) if glossary.owner is not None else ""
)
glossary_followers = []
if glossary.followers:
for follower in glossary.followers.__root__:
glossary_followers.append(str(follower.id.__root__))
tier = None
for glossary_tag in glossary.tags:
if "Tier" in glossary_tag.tagFQN:
tier = glossary_tag.tagFQN
else:
tags.add(glossary_tag.tagFQN)
# tasks: List[Task] = glossary.tasks # TODO Handle Glossary words
# task_names = []
# task_descriptions = []
# for task in tasks:
# task_names.append(task.displayName)
# if task.description is not None:
# task_descriptions.append(task.description)
# if tags in task and len(task.tags) > 0:
# for col_tag in task.tags:
# tags.add(col_tag.tagFQN)
glossary_doc = GlossaryESDocument(
glossary_id=str(glossary.id.__root__),
glossary_name=glossary.displayName
if glossary.displayName
else glossary.name,
# task_names=task_names, # TODO Handle Glossary words
# task_descriptions=task_descriptions,
suggest=suggest,
description=glossary.description,
last_updated_timestamp=timestamp,
tier=tier,
tags=list(tags),
fqdn=fqdn,
owner=glossary_owner,
followers=glossary_followers,
)
return glossary_doc
def _get_charts(self, chart_refs: Optional[List[entityReference.EntityReference]]): def _get_charts(self, chart_refs: Optional[List[entityReference.EntityReference]]):
charts = [] charts = []
if chart_refs: if chart_refs:

View File

@ -200,6 +200,48 @@ TOPIC_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
""" """
) )
GLOSSARY_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
"""
{
"mappings":{
"properties": {
"glossary_name": {
"type":"text"
},
"display_name": {
"type": "text"
},
"owner": {
"type": "text"
},
"followers": {
"type": "keyword"
},
"last_updated_timestamp": {
"type": "date",
"format": "epoch_second"
},
"description": {
"type": "text"
},
"tier": {
"type": "keyword"
},
"tags": {
"type": "keyword"
},
"entity_type": {
"type": "keyword"
},
"suggest": {
"type": "completion"
}
}
}
}
"""
)
DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
""" """
{ {

View File

@ -23,6 +23,7 @@ from metadata.ingestion.api.common import Entity, WorkflowContext
from metadata.ingestion.api.source import Source, SourceStatus from metadata.ingestion.api.source import Source, SourceStatus
from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
from metadata.ingestion.ometa.openmetadata_rest import Glossary
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -33,6 +34,7 @@ class MetadataTablesRestSourceConfig(ConfigModel):
include_tables: Optional[bool] = True include_tables: Optional[bool] = True
include_topics: Optional[bool] = True include_topics: Optional[bool] = True
include_dashboards: Optional[bool] = True include_dashboards: Optional[bool] = True
include_glossary: Optional[bool] = True
include_pipelines: Optional[bool] = True include_pipelines: Optional[bool] = True
limit_records: int = 1000 limit_records: int = 1000
@ -60,6 +62,10 @@ class MetadataSourceStatus(SourceStatus):
self.success.append(table_name) self.success.append(table_name)
logger.info("Table Scanned: %s", table_name) logger.info("Table Scanned: %s", table_name)
def scanned_glossary(self, glossary_name: str) -> None:
self.success.append(glossary_name)
logger.info("Glossary Scanned: {}".format(glossary_name))
def scanned_topic(self, topic_name: str) -> None: def scanned_topic(self, topic_name: str) -> None:
"""scanned topic method """scanned topic method
@ -143,6 +149,7 @@ class MetadataSource(Source[Entity]):
def next_record(self) -> Iterable[Entity]: def next_record(self) -> Iterable[Entity]:
yield from self.fetch_table() yield from self.fetch_table()
yield from self.fetch_topic() yield from self.fetch_topic()
yield from self.fetch_glossary()
yield from self.fetch_dashboard() yield from self.fetch_dashboard()
yield from self.fetch_pipeline() yield from self.fetch_pipeline()
@ -247,6 +254,23 @@ class MetadataSource(Source[Entity]):
break break
after = pipeline_entities.after after = pipeline_entities.after
def fetch_glossary(self) -> Glossary:
if self.config.include_glossary:
after = None
while True:
glossary_entities = self.metadata.list_entities(
entity=Glossary,
fields=["owner", "tags", "followers"],
after=after,
limit=self.config.limit_records,
)
for glossary in glossary_entities.entities:
self.status.scanned_glossary(glossary.name)
yield glossary
if glossary_entities.after is None:
break
after = glossary_entities.after
def get_status(self) -> SourceStatus: def get_status(self) -> SourceStatus:
return self.status return self.status

View File

@ -27,6 +27,7 @@ from metadata.generated.schema.entity.data.mlmodel import MlModel
from metadata.generated.schema.entity.data.pipeline import Pipeline from metadata.generated.schema.entity.data.pipeline import Pipeline
from metadata.generated.schema.entity.data.report import Report from metadata.generated.schema.entity.data.report import Report
from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.data.table import Table
from metadata.generated.schema.entity.data.glossary import Glossary
from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.data.topic import Topic
from metadata.generated.schema.entity.services.dashboardService import DashboardService from metadata.generated.schema.entity.services.dashboardService import DashboardService
from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.entity.services.databaseService import DatabaseService
@ -51,6 +52,10 @@ class OMetaEndpointTest(TestCase):
Pass Entities and test their suffix generation Pass Entities and test their suffix generation
""" """
# Glossary
self.assertEqual(self.metadata.get_suffix(Glossary), "/glossary")
# ML # ML
self.assertEqual(self.metadata.get_suffix(MlModel), "/mlmodels") self.assertEqual(self.metadata.get_suffix(MlModel), "/mlmodels")