diff --git a/bootstrap/sql/mysql/v004__create_db_connection_info.sql b/bootstrap/sql/mysql/v004__create_db_connection_info.sql new file mode 100644 index 00000000000..1da545a8106 --- /dev/null +++ b/bootstrap/sql/mysql/v004__create_db_connection_info.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS glossary_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')), + timestamp BIGINT, + PRIMARY KEY (id), + UNIQUE KEY unique_name(fullyQualifiedName), + INDEX (updatedBy), + INDEX (updatedAt) +); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java index 0d3fdda73ee..d0ade3efdeb 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java @@ -70,6 +70,7 @@ public final class Entity { public static final String UNUSED = "unused"; public static final String BOTS = "bots"; public static final String LOCATION = "location"; + public static final String GLOSSARY = "glossary"; // // Policies diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java index 78455f3318c..cc95cb1e8b4 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Collectors; +import javax.ws.rs.core.Response; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; @@ -31,6 +32,7 @@ import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.common.xcontent.XContentType; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.entity.data.Dashboard; +import org.openmetadata.catalog.entity.data.Glossary; import org.openmetadata.catalog.entity.data.Pipeline; import org.openmetadata.catalog.entity.data.Table; import org.openmetadata.catalog.entity.data.Topic; @@ -63,7 +65,8 @@ public class ElasticSearchIndexDefinition { TABLE_SEARCH_INDEX("table_search_index", "/elasticsearch/table_index_mapping.json"), TOPIC_SEARCH_INDEX("topic_search_index", "/elasticsearch/topic_index_mapping.json"), DASHBOARD_SEARCH_INDEX("dashboard_search_index", "/elasticsearch/dashboard_index_mapping.json"), - PIPELINE_SEARCH_INDEX("pipeline_search_index", "/elasticsearch/pipeline_index_mapping.json"); + PIPELINE_SEARCH_INDEX("pipeline_search_index", "/elasticsearch/pipeline_index_mapping.json"), + GLOSSARY_SEARCH_INDEX("glossary_search_index", "/elasticsearch/glossary_index_mapping.json"); public final String indexName; public final String indexMappingFile; @@ -189,6 +192,8 @@ public class ElasticSearchIndexDefinition { return ElasticSearchIndexType.PIPELINE_SEARCH_INDEX; } else if (type.equalsIgnoreCase(Entity.TOPIC)) { return ElasticSearchIndexType.TOPIC_SEARCH_INDEX; + } else if (type.equalsIgnoreCase(Entity.TOPIC)) { + return ElasticSearchIndexType.GLOSSARY_SEARCH_INDEX; } throw new RuntimeException("Failed to find index doc for type " + type); } @@ -707,3 +712,68 @@ class PipelineESIndex extends ElasticSearchIndex { return pipelineESIndexBuilder; } } + +@EqualsAndHashCode(callSuper = true) +@Getter +@SuperBuilder(builderMethodName = "internalBuilder") +@Value +@JsonInclude(JsonInclude.Include.NON_NULL) +class GlossaryESIndex extends ElasticSearchIndex { + @JsonProperty("glossary_id") + String glossaryId; + + public static GlossaryESIndexBuilder builder(Glossary glossary, int responseCode) { + List tags = new ArrayList<>(); + List taskNames = new ArrayList<>(); + List taskDescriptions = new ArrayList<>(); + List suggest = new ArrayList<>(); + suggest.add(ElasticSearchSuggest.builder().input(glossary.getFullyQualifiedName()).weight(5).build()); + suggest.add(ElasticSearchSuggest.builder().input(glossary.getDisplayName()).weight(10).build()); + + if (glossary.getTags() != null) { + glossary.getTags().forEach(tag -> tags.add(tag.getTagFQN())); + } + + Long updatedTimestamp = glossary.getUpdatedAt(); + ParseTags parseTags = new ParseTags(tags); + String description = glossary.getDescription() != null ? glossary.getDescription() : ""; + String displayName = glossary.getDisplayName() != null ? glossary.getDisplayName() : ""; + GlossaryESIndexBuilder builder = + internalBuilder() + .glossaryId(glossary.getId().toString()) + .name(glossary.getDisplayName()) + .displayName(description) + .description(displayName) + .fqdn(glossary.getFullyQualifiedName()) + .lastUpdatedTimestamp(updatedTimestamp) + .entityType("glossary") + .suggest(suggest) + .tags(parseTags.tags) + .tier(parseTags.tierTag); + + if (glossary.getFollowers() != null) { + builder.followers( + glossary.getFollowers().stream().map(item -> item.getId().toString()).collect(Collectors.toList())); + } else if (responseCode == Response.Status.CREATED.getStatusCode()) { + builder.followers(Collections.emptyList()); + } + + if (glossary.getOwner() != null) { + builder.owner(glossary.getOwner().getId().toString()); + } + + ESChangeDescription esChangeDescription = null; + if (glossary.getChangeDescription() != null) { + esChangeDescription = + ESChangeDescription.builder().updatedAt(updatedTimestamp).updatedBy(glossary.getUpdatedBy()).build(); + esChangeDescription.setFieldsAdded(glossary.getChangeDescription().getFieldsAdded()); + esChangeDescription.setFieldsDeleted(glossary.getChangeDescription().getFieldsDeleted()); + esChangeDescription.setFieldsUpdated(glossary.getChangeDescription().getFieldsUpdated()); + } else if (responseCode == Response.Status.CREATED.getStatusCode()) { + esChangeDescription = + ESChangeDescription.builder().updatedAt(updatedTimestamp).updatedBy(glossary.getUpdatedBy()).build(); + } + builder.changeDescriptions(esChangeDescription != null ? List.of(esChangeDescription) : null); + return builder; + } +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java index b840f9d023c..e5a0f3ea42c 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java @@ -33,6 +33,7 @@ import org.openmetadata.catalog.entity.Bots; import org.openmetadata.catalog.entity.data.Chart; import org.openmetadata.catalog.entity.data.Dashboard; import org.openmetadata.catalog.entity.data.Database; +import org.openmetadata.catalog.entity.data.Glossary; import org.openmetadata.catalog.entity.data.Location; import org.openmetadata.catalog.entity.data.Metrics; import org.openmetadata.catalog.entity.data.MlModel; @@ -58,6 +59,7 @@ import org.openmetadata.catalog.jdbi3.DashboardRepository.DashboardEntityInterfa import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceEntityInterface; import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface; import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface; +import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface; import org.openmetadata.catalog.jdbi3.LocationRepository.LocationEntityInterface; import org.openmetadata.catalog.jdbi3.MessagingServiceRepository.MessagingServiceEntityInterface; import org.openmetadata.catalog.jdbi3.MetricsRepository.MetricsEntityInterface; @@ -134,6 +136,9 @@ public interface CollectionDAO { @CreateSqlObject MlModelDAO mlModelDAO(); + @CreateSqlObject + GlossaryDAO glossaryDAO(); + @CreateSqlObject BotsDAO botsDAO(); @@ -689,6 +694,28 @@ public interface CollectionDAO { } } + interface GlossaryDAO extends EntityDAO { + @Override + default String getTableName() { + return "glossary_entity"; + } + + @Override + default Class getEntityClass() { + return Glossary.class; + } + + @Override + default String getNameColumn() { + return "fullyQualifiedName"; + } + + @Override + default EntityReference getEntityReference(Glossary entity) { + return new GlossaryEntityInterface(entity).getEntityReference(); + } + } + interface AirflowPipelineDAO extends EntityDAO { @Override default String getTableName() { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java new file mode 100644 index 00000000000..8c10f2f1118 --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.jdbi3; + +import java.io.IOException; +import java.net.URI; +import java.text.ParseException; +import java.util.List; +import java.util.UUID; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.entity.data.Glossary; +import org.openmetadata.catalog.resources.glossary.GlossaryResource; +import org.openmetadata.catalog.type.ChangeDescription; +import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.type.TagLabel; +import org.openmetadata.catalog.util.EntityInterface; +import org.openmetadata.catalog.util.EntityUtil; +import org.openmetadata.catalog.util.EntityUtil.Fields; +import org.openmetadata.catalog.util.JsonUtils; + +public class GlossaryRepository extends EntityRepository { + private static final Fields GLOSSARY_UPDATE_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "owner,tags"); + private static final Fields GLOSSARY_PATCH_FIELDS = new Fields(GlossaryResource.FIELD_LIST, "owner,tags"); + private final CollectionDAO dao; + + public GlossaryRepository(CollectionDAO dao) { + super( + GlossaryResource.COLLECTION_PATH, + Entity.GLOSSARY, + Glossary.class, + dao.glossaryDAO(), + dao, + GLOSSARY_PATCH_FIELDS, + GLOSSARY_UPDATE_FIELDS, + true, + true, + false); + this.dao = dao; + } + + public static String getFQN(Glossary glossary) { + return (glossary.getName()); + } + + @Transaction + public EntityReference getOwnerReference(Glossary glossary) throws IOException { + return EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), glossary.getOwner()); + } + + @Override + public Glossary setFields(Glossary glossary, Fields fields) throws IOException, ParseException { + glossary.setDisplayName(glossary.getDisplayName()); + glossary.setOwner(fields.contains("owner") ? getOwner(glossary) : null); + glossary.setFollowers(fields.contains("followers") ? getFollowers(glossary) : null); + glossary.setTags(fields.contains("tags") ? getTags(glossary.getFullyQualifiedName()) : null); + glossary.setUsageSummary( + fields.contains("usageSummary") ? EntityUtil.getLatestUsage(dao.usageDAO(), glossary.getId()) : null); + return glossary; + } + + @Override + public void prepare(Glossary glossary) throws IOException { + glossary.setFullyQualifiedName(getFQN(glossary)); + EntityUtil.populateOwner(dao.userDAO(), dao.teamDAO(), glossary.getOwner()); // Validate owner + glossary.setTags(EntityUtil.addDerivedTags(dao.tagDAO(), glossary.getTags())); + } + + @Override + public void storeEntity(Glossary glossary, boolean update) throws IOException { + // Relationships and fields such as href are derived and not stored as part of json + EntityReference owner = glossary.getOwner(); + List tags = glossary.getTags(); + + // Don't store owner, dashboard, href and tags as JSON. Build it on the fly based on relationships + glossary.withOwner(null).withHref(null).withTags(null); + + if (update) { + dao.glossaryDAO().update(glossary.getId(), JsonUtils.pojoToJson(glossary)); + } else { + dao.glossaryDAO().insert(glossary); + } + + // Restore the relationships + glossary.withOwner(owner).withTags(tags); + } + + @Override + public void restorePatchAttributes(Glossary original, Glossary updated) {} + + @Override + public EntityInterface getEntityInterface(Glossary entity) { + return new GlossaryEntityInterface(entity); + } + + @Override + public void storeRelationships(Glossary glossary) { + setOwner(glossary, glossary.getOwner()); + applyTags(glossary); + } + + @Override + public EntityUpdater getUpdater(Glossary original, Glossary updated, Operation operation) { + return new GlossaryUpdater(original, updated, operation); + } + + public static class GlossaryEntityInterface implements EntityInterface { + private final Glossary entity; + + public GlossaryEntityInterface(Glossary entity) { + this.entity = entity; + } + + @Override + public UUID getId() { + return entity.getId(); + } + + @Override + public String getDescription() { + return entity.getDescription(); + } + + @Override + public String getDisplayName() { + return entity.getDisplayName(); + } + + @Override + public Boolean isDeleted() { + return entity.getDeleted(); + } + + @Override + public EntityReference getOwner() { + return entity.getOwner(); + } + + @Override + public String getFullyQualifiedName() { + return entity.getFullyQualifiedName(); + } + + @Override + public List getTags() { + return entity.getTags(); + } + + public String getSkos() { + return entity.getSkos(); + } + + @Override + public Double getVersion() { + return entity.getVersion(); + } + + @Override + public String getUpdatedBy() { + return entity.getUpdatedBy(); + } + + @Override + public long getUpdatedAt() { + return entity.getUpdatedAt(); + } + + @Override + public URI getHref() { + return entity.getHref(); + } + + @Override + public List getFollowers() { + return entity.getFollowers(); + } + + @Override + public ChangeDescription getChangeDescription() { + return entity.getChangeDescription(); + } + + @Override + public EntityReference getEntityReference() { + return new EntityReference() + .withId(getId()) + .withName(getFullyQualifiedName()) + .withDescription(getDescription()) + .withDisplayName(getDisplayName()) + .withType(Entity.GLOSSARY); + } + + @Override + public Glossary getEntity() { + return entity; + } + + @Override + public EntityReference getContainer() { + return null; + } + + @Override + public void setId(UUID id) { + entity.setId(id); + } + + @Override + public void setDescription(String description) { + entity.setDescription(description); + } + + @Override + public void setDisplayName(String displayName) { + entity.setDisplayName(displayName); + } + + @Override + public void setUpdateDetails(String updatedBy, long updatedAt) { + entity.setUpdatedBy(updatedBy); + entity.setUpdatedAt(updatedAt); + } + + @Override + public void setChangeDescription(Double newVersion, ChangeDescription changeDescription) { + entity.setVersion(newVersion); + entity.setChangeDescription(changeDescription); + } + + @Override + public void setOwner(EntityReference owner) { + entity.setOwner(owner); + } + + @Override + public void setDeleted(boolean flag) { + entity.setDeleted(flag); + } + + @Override + public Glossary withHref(URI href) { + return entity.withHref(href); + } + + @Override + public void setTags(List tags) { + entity.setTags(tags); + } + } + + /** Handles entity updated from PUT and POST operation. */ + public class GlossaryUpdater extends EntityUpdater { + public GlossaryUpdater(Glossary original, Glossary updated, Operation operation) { + super(original, updated, operation); + } + } +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java new file mode 100644 index 00000000000..28381a6ecba --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/glossary/GlossaryResource.java @@ -0,0 +1,436 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.glossary; + +import com.google.inject.Inject; +import io.swagger.annotations.Api; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.security.GeneralSecurityException; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import javax.json.JsonPatch; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.PATCH; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.api.data.CreateGlossary; +import org.openmetadata.catalog.entity.data.Glossary; +import org.openmetadata.catalog.jdbi3.CollectionDAO; +import org.openmetadata.catalog.jdbi3.GlossaryRepository; +import org.openmetadata.catalog.resources.Collection; +import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.security.SecurityUtil; +import org.openmetadata.catalog.type.EntityHistory; +import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.type.Include; +import org.openmetadata.catalog.util.EntityUtil.Fields; +import org.openmetadata.catalog.util.RestUtil; +import org.openmetadata.catalog.util.RestUtil.DeleteResponse; +import org.openmetadata.catalog.util.RestUtil.PatchResponse; +import org.openmetadata.catalog.util.RestUtil.PutResponse; +import org.openmetadata.catalog.util.ResultList; + +@Path("/v1/glossary") +@Api(value = "Glossary collection", tags = "Glossary collection") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "glossary") +public class GlossaryResource { + public static final String COLLECTION_PATH = "v1/glossary/"; + private final GlossaryRepository dao; + private final Authorizer authorizer; + + public static void addHref(UriInfo uriInfo, EntityReference ref) { + ref.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, ref.getId())); + } + + public static List addHref(UriInfo uriInfo, List glossary) { + Optional.ofNullable(glossary).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i)); + return glossary; + } + + public static Glossary addHref(UriInfo uriInfo, Glossary glossary) { + glossary.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, glossary.getId())); + Entity.withHref(uriInfo, glossary.getOwner()); + Entity.withHref(uriInfo, glossary.getFollowers()); + return glossary; + } + + @Inject + public GlossaryResource(CollectionDAO dao, Authorizer authorizer) { + Objects.requireNonNull(dao, "GlossaryRepository must not be null"); + this.dao = new GlossaryRepository(dao); + this.authorizer = authorizer; + } + + public static class GlossaryList extends ResultList { + @SuppressWarnings("unused") + GlossaryList() { + // Empty constructor needed for deserialization + } + + public GlossaryList(List data, String beforeCursor, String afterCursor, int total) + throws GeneralSecurityException, UnsupportedEncodingException { + super(data, beforeCursor, afterCursor, total); + } + } + + static final String FIELDS = "owner,dashboard,definition,followers,tags,usageSummary,skos"; + public static final List FIELD_LIST = Arrays.asList(FIELDS.replaceAll(" ", "").split(",")); + + @GET + @Valid + @Operation( + summary = "List Glossary", + tags = "glossary", + description = + "Get a list of glossary. Use `fields` parameter to get only necessary fields. " + + " Use cursor-based pagination to limit the number " + + "entries in the list using `limit` and `before` or `after` query params.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of glossary", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter(description = "Limit the number glossary returned. (1 to 1000000, " + "default = 10)") + @DefaultValue("10") + @Min(1) + @Max(1000000) + @QueryParam("limit") + int limitParam, + @Parameter(description = "Returns list of glossary before this cursor", schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter(description = "Returns list of glossary after this cursor", schema = @Schema(type = "string")) + @QueryParam("after") + String after, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException, GeneralSecurityException, ParseException { + RestUtil.validateCursors(before, after); + Fields fields = new Fields(FIELD_LIST, fieldsParam); + + ResultList glossary; + if (before != null) { // Reverse paging + glossary = dao.listBefore(uriInfo, fields, null, limitParam, before, include); // Ask for one extra entry + } else { // Forward paging or first page + glossary = dao.listAfter(uriInfo, fields, null, limitParam, after, include); + } + addHref(uriInfo, glossary.getData()); + return glossary; + } + + @GET + @Path("/{id}") + @Operation( + summary = "Get a glossary", + tags = "glossary", + description = "Get a glossary by `id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The glossary", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))), + @ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found") + }) + public Glossary get( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") String id, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException, ParseException { + Fields fields = new Fields(FIELD_LIST, fieldsParam); + return addHref(uriInfo, dao.get(uriInfo, id, fields, include)); + } + + @GET + @Path("/name/{fqn}") + @Operation( + summary = "Get a glossary by name", + tags = "glossary", + description = "Get a glossary by fully qualified name.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The glossary", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))), + @ApiResponse(responseCode = "404", description = "Glossary for instance {id} is not found") + }) + public Glossary getByName( + @Context UriInfo uriInfo, + @PathParam("fqn") String fqn, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException, ParseException { + Fields fields = new Fields(FIELD_LIST, fieldsParam); + Glossary glossary = dao.getByName(uriInfo, fqn, fields, include); + return addHref(uriInfo, glossary); + } + + @GET + @Path("/{id}/versions") + @Operation( + summary = "List glossary versions", + tags = "glossary", + description = "Get a list of all the versions of a glossary identified by `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of glossary versions", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class))) + }) + public EntityHistory listVersions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id) + throws IOException, ParseException { + return dao.listVersions(id); + } + + @GET + @Path("/{id}/versions/{version}") + @Operation( + summary = "Get a version of the glossary", + tags = "glossary", + description = "Get a version of the glossary by given `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "glossary", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))), + @ApiResponse( + responseCode = "404", + description = "Glossary for instance {id} and version {version} is " + "not found") + }) + public Glossary getVersion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "glossary Id", schema = @Schema(type = "string")) @PathParam("id") String id, + @Parameter( + description = "glossary version number in the form `major`.`minor`", + schema = @Schema(type = "string", example = "0.1 or 1.1")) + @PathParam("version") + String version) + throws IOException, ParseException { + return dao.getVersion(id, version); + } + + @POST + @Operation( + summary = "Create a glossary", + tags = "glossary", + description = "Create a new glossary.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The glossary", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = CreateGlossary.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response create( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create) + throws IOException, ParseException { + SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + Glossary glossary = getGlossary(securityContext, create); + glossary = addHref(uriInfo, dao.create(uriInfo, glossary)); + return Response.created(glossary.getHref()).entity(glossary).build(); + } + + @PATCH + @Path("/{id}") + @Operation( + summary = "Update a glossary", + tags = "glossary", + description = "Update an existing glossary using JsonPatch.", + externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902")) + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + public Response updateDescription( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") String id, + @RequestBody( + description = "JsonPatch with array of operations", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON_PATCH_JSON, + examples = { + @ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]") + })) + JsonPatch patch) + throws IOException, ParseException { + Fields fields = new Fields(FIELD_LIST, FIELDS); + Glossary glossary = dao.get(uriInfo, id, fields); + SecurityUtil.checkAdminRoleOrPermissions( + authorizer, securityContext, dao.getEntityInterface(glossary).getEntityReference(), patch); + PatchResponse response = + dao.patch(uriInfo, UUID.fromString(id), securityContext.getUserPrincipal().getName(), patch); + addHref(uriInfo, response.getEntity()); + return response.toResponse(); + } + + @PUT + @Operation( + summary = "Create or update a glossary", + tags = "glossary", + description = "Create a new glossary, if it does not exist or update an existing glossary.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The glossary", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Glossary.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response createOrUpdate( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create) + throws IOException, ParseException { + Glossary glossary = getGlossary(securityContext, create); + PutResponse response = dao.createOrUpdate(uriInfo, glossary); + addHref(uriInfo, response.getEntity()); + return response.toResponse(); + } + + @PUT + @Path("/{id}/followers") + @Operation( + summary = "Add a follower", + tags = "glossary", + description = "Add a user identified by `userId` as follower of this glossary", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "glossary for instance {id} is not found") + }) + public Response addFollower( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the glossary", schema = @Schema(type = "string")) @PathParam("id") String id, + @Parameter(description = "Id of the user to be added as follower", schema = @Schema(type = "string")) + String userId) + throws IOException, ParseException { + return dao.addFollower(securityContext.getUserPrincipal().getName(), UUID.fromString(id), UUID.fromString(userId)) + .toResponse(); + } + + @DELETE + @Path("/{id}/followers/{userId}") + @Operation( + summary = "Remove a follower", + tags = "glossary", + description = "Remove the user identified `userId` as a follower of the glossary.") + public Response deleteFollower( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the glossary", schema = @Schema(type = "string")) @PathParam("id") String id, + @Parameter(description = "Id of the user being removed as follower", schema = @Schema(type = "string")) + @PathParam("userId") + String userId) + throws IOException, ParseException { + return dao.deleteFollower( + securityContext.getUserPrincipal().getName(), UUID.fromString(id), UUID.fromString(userId)) + .toResponse(); + } + + @DELETE + @Path("/{id}") + @Operation( + summary = "Delete a Glossary", + tags = "glossary", + description = "Delete a glossary by `id`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "glossary for instance {id} is not found") + }) + public Response delete(@Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("id") String id) + throws IOException, ParseException { + SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + DeleteResponse response = dao.delete(securityContext.getUserPrincipal().getName(), id); + return response.toResponse(); + } + + private Glossary getGlossary(SecurityContext securityContext, CreateGlossary create) { + return new Glossary() + .withId(UUID.randomUUID()) + .withName(create.getName()) + .withDisplayName(create.getDisplayName()) + .withDescription(create.getDescription()) + .withSkos(create.getSkos()) + .withTags(create.getTags()) + .withOwner(create.getOwner()) + .withUpdatedBy(securityContext.getUserPrincipal().getName()) + .withUpdatedAt(System.currentTimeMillis()); + } +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java index 00b3986bacb..eeb77e57b98 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java @@ -146,6 +146,9 @@ public class SearchResource { case "table_search_index": searchSourceBuilder = buildTableSearchBuilder(query, from, size); break; + case "glossary_search_index": + searchSourceBuilder = buildGlossarySearchBuilder(query, from, size); + break; default: searchSourceBuilder = buildAggregateSearchBuilder(query, from, size); break; @@ -353,4 +356,27 @@ public class SearchResource { return searchSourceBuilder; } + + private SearchSourceBuilder buildGlossarySearchBuilder(String query, int from, int size) { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + HighlightBuilder.Field highlightGlossaryName = new HighlightBuilder.Field("glossary_name"); + highlightGlossaryName.highlighterType("unified"); + HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field("description"); + highlightDescription.highlighterType("unified"); + HighlightBuilder hb = new HighlightBuilder(); + hb.field(highlightDescription); + hb.field(highlightGlossaryName); + hb.preTags(""); + hb.postTags(""); + searchSourceBuilder + .query(QueryBuilders.queryStringQuery(query).field("glossary_name", 5.0f).field("description").lenient(true)) + .aggregation(AggregationBuilders.terms("EntityType").field("entity_type")) + .aggregation(AggregationBuilders.terms("Tier").field("tier")) + .aggregation(AggregationBuilders.terms("Tags").field("tags")) + .highlighter(hb) + .from(from) + .size(size); + + return searchSourceBuilder; + } } diff --git a/catalog-rest-service/src/main/resources/json/schema/api/data/createGlossary.json b/catalog-rest-service/src/main/resources/json/schema/api/data/createGlossary.json new file mode 100644 index 00000000000..0a7a3fb9de0 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/api/data/createGlossary.json @@ -0,0 +1,40 @@ +{ + "$id": "https://open-metadata.org/schema/api/data/createGlossary.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Create Glossary entity request", + "description": "Create Glossary entity request", + "type": "object", + "properties" : { + "name": { + "description": "Name that identifies this glossary.", + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "displayName": { + "description": "Display Name that identifies this glossary.", + "type": "string" + }, + "description": { + "description": "Description of the glossary instance.", + "type": "string" + }, + "skos": { + "description": "SKOS data in JSON-LD format", + "type": "string" + }, + "tags": { + "description": "Tags for this glossary", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "owner": { + "description": "Owner of this glossary", + "$ref": "../../type/entityReference.json" + } + }, + "required": ["name"] + } \ No newline at end of file diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/glossary.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/glossary.json new file mode 100644 index 00000000000..bb16bd5e790 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/glossary.json @@ -0,0 +1,85 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/glossary.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Glossary", + "description": "This schema defines the Glossary entity based on SKOS.", + "type": "object", + + "properties" : { + "id": { + "description": "Unique identifier of a glossary instance.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name that identifies this glossary.", + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "fullyQualifiedName": { + "description": "A unique name that identifies a glossary.", + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "displayName": { + "description": "Display Name that identifies this glossary.", + "type": "string" + }, + "version" : { + "description": "Metadata version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/entityVersion" + }, + "updatedAt" : { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy" : { + "description": "User who made the update.", + "type": "string" + }, + "description": { + "description": "Description of the glossary.", + "type": "string" + }, + "skos": { + "description": "SKOS data in JSON-LD format", + "type": "string" + }, + "href": { + "description": "Link to the resource corresponding to this entity.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "owner": { + "description": "Owner of this glossary.", + "$ref": "../../type/entityReference.json" + }, + "followers": { + "description": "Followers of this glossary.", + "$ref": "../../type/entityReference.json#/definitions/entityReferenceList" + }, + "tags": { + "description": "Tags for this glossary.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "usageSummary" : { + "description": "Latest usage information for this glossary.", + "$ref": "../../type/usageDetails.json", + "default": null + }, + "changeDescription": { + "description" : "Change that lead to this version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "deleted" : { + "description": "When `true` indicates the entity has been soft deleted.", + "type" : "boolean", + "default": false + } + }, + "required": ["id", "name"] + } \ No newline at end of file diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryResourceTest.java new file mode 100644 index 00000000000..43c32b7a210 --- /dev/null +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/glossary/GlossaryResourceTest.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.glossary; + +import static javax.ws.rs.core.Response.Status.FORBIDDEN; +import static javax.ws.rs.core.Response.Status.NOT_FOUND; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.openmetadata.catalog.security.SecurityUtil.authHeaders; +import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS; +import static org.openmetadata.catalog.util.TestUtils.NON_EXISTENT_ENTITY; +import static org.openmetadata.catalog.util.TestUtils.assertResponse; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Map; +import java.util.UUID; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestMethodOrder; +import org.openmetadata.catalog.CatalogApplicationTest; +import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.api.data.CreateGlossary; +import org.openmetadata.catalog.entity.data.Glossary; +import org.openmetadata.catalog.exception.CatalogExceptionMessage; +import org.openmetadata.catalog.jdbi3.GlossaryRepository.GlossaryEntityInterface; +import org.openmetadata.catalog.resources.EntityResourceTest; +import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.util.TestUtils; + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class GlossaryResourceTest extends EntityResourceTest { + public GlossaryResourceTest() { + super( + Entity.GLOSSARY, + Glossary.class, + GlossaryResource.GlossaryList.class, + "glossary", + GlossaryResource.FIELDS, + false, + true, + true, + true); + } + + @BeforeAll + public void setup(TestInfo test) throws IOException, URISyntaxException { + super.setup(test); + } + + @Override + public CreateGlossary createRequest(String name, String description, String displayName, EntityReference owner) { + return new CreateGlossary() + .withName(name) + .withDescription(description) + .withDisplayName(displayName) + .withOwner(owner); + } + + @Override + public EntityReference getContainer(CreateGlossary createRequest) { + return null; + } + + // TODO: Entity tests + + // @Test + // public void patch_entityAttributes_200_ok() + // {} + + // @Test + // public void put_entityCreate_200() + // {} + + // @Test + // public void put_entityCreate_as_owner_200() + // {} + + // @Test + // public void put_entityEmptyDescriptionUpdate_200() + // {} + + // @Test + // public void put_entityNullDescriptionUpdate_200() + // {} + + // @Test + // public void put_entityUpdateOwner_200() + // {} + + // @Test + // public void put_entityUpdateWithNoChange_200() + // {} + + @Test + public void post_validGlossary_200_OK(TestInfo test) throws IOException { + CreateGlossary create = create(test).withDescription("description"); + createAndCheckEntity(create, ADMIN_AUTH_HEADERS); + } + + @Test + public void post_glossaryWithUserOwner_200_ok(TestInfo test) throws IOException { + createAndCheckEntity(create(test).withOwner(USER_OWNER1), ADMIN_AUTH_HEADERS); + } + + @Test + public void post_glossaryWithTeamOwner_200_ok(TestInfo test) throws IOException { + createAndCheckEntity(create(test).withOwner(TEAM_OWNER1), ADMIN_AUTH_HEADERS); + } + + // @Test + // public void post_glossaryWithInvalidOwnerType_4xx(TestInfo test) { + // EntityReference owner = new EntityReference().withId(TEAM1.getId()); /* No owner type is set */ + // CreateGlossary create = create(test).withOwner(owner); + // HttpResponseException exception = assertThrows(HttpResponseException.class, () -> + // createEntity(create, ADMIN_AUTH_HEADERS)); + // TestUtils.assertResponseContains(exception, BAD_REQUEST, "type must not be null"); + // } + + @Test + public void post_glossary_as_non_admin_401(TestInfo test) { + CreateGlossary create = create(test); + HttpResponseException exception = + assertThrows(HttpResponseException.class, () -> createEntity(create, authHeaders("test@open-metadata.org"))); + assertResponse(exception, FORBIDDEN, "Principal: CatalogPrincipal{name='test'} is not admin"); + } + + @Test + public void get_nonExistentGlossary_404_notFound() { + HttpResponseException exception = + assertThrows(HttpResponseException.class, () -> getEntity(NON_EXISTENT_ENTITY, ADMIN_AUTH_HEADERS)); + assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, NON_EXISTENT_ENTITY)); + } + + @Test + public void delete_glossary_200_ok(TestInfo test) throws HttpResponseException { + Glossary glossary = createEntity(create(test), ADMIN_AUTH_HEADERS); + deleteGlossary(glossary.getId(), ADMIN_AUTH_HEADERS); + } + + // @Test + // public void delete_glossary_as_non_admin_401(TestInfo test) throws HttpResponseException { + // Glossary glossary = createEntity(create(test), ADMIN_AUTH_HEADERS); + // HttpResponseException exception = assertThrows(HttpResponseException.class, () -> + // deleteGlossary(glossary.getId(), authHeaders("test@open-metadata.org"))); + // assertResponse(exception, FORBIDDEN, "Principal: CatalogPrincipal{name='test'} is not admin"); + // } + + @Test + public void delete_nonExistentGlossary_404() { + HttpResponseException exception = + assertThrows(HttpResponseException.class, () -> getEntity(NON_EXISTENT_ENTITY, ADMIN_AUTH_HEADERS)); + assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, NON_EXISTENT_ENTITY)); + } + + // @Test + // public void put_GlossaryCreate_200(TestInfo test) throws HttpResponseException { + // // Create a new Glossary with PUT + // CreateGlossary request = create(test).withOwner(USER_OWNER1); + // updateAndCheckGlossary(null, request.withName(test.getDisplayName()).withDescription(null), CREATED, + // ADMIN_AUTH_HEADERS, NO_CHANGE); + // } + + public static CreateGlossary create(TestInfo test) { + return create(test, 0); + } + + public static CreateGlossary create(TestInfo test, int index) { + return new CreateGlossary().withName(getGlossaryName(test, index)); + } + + /** + * A method variant to be called form other tests to create a glossary without depending on Database, DatabaseService + * set up in the {@code setup()} method + */ + public Glossary createEntity(TestInfo test, int index) throws IOException { + CreateGlossary create = new CreateGlossary().withName(getGlossaryName(test, index)); + return createEntity(create, ADMIN_AUTH_HEADERS); + } + + private void deleteGlossary(UUID id, Map authHeaders) throws HttpResponseException { + TestUtils.delete(CatalogApplicationTest.getResource("glossary/" + id), authHeaders); + + // Check to make sure database entity does not exist + HttpResponseException exception = assertThrows(HttpResponseException.class, () -> getEntity(id, authHeaders)); + assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, id)); + } + + public static String getGlossaryName(TestInfo test, int index) { + return String.format("glossary%d_%s", index, test.getDisplayName()); + } + + @Override + public void validateCreatedEntity( + Glossary createdEntity, CreateGlossary createRequest, Map authHeaders) + throws HttpResponseException { + validateCommonEntityFields( + getEntityInterface(createdEntity), + createRequest.getDescription(), + TestUtils.getPrincipal(authHeaders), + createRequest.getOwner()); + + // Entity specific validation + TestUtils.validateTags(createRequest.getTags(), createdEntity.getTags()); + TestUtils.validateEntityReference(createdEntity.getFollowers()); + } + + @Override + public void validateUpdatedEntity(Glossary updated, CreateGlossary request, Map authHeaders) + throws HttpResponseException { + validateCreatedEntity(updated, request, authHeaders); + } + + @Override + public void compareEntities(Glossary expected, Glossary patched, Map authHeaders) + throws HttpResponseException { + validateCommonEntityFields( + getEntityInterface(patched), + expected.getDescription(), + TestUtils.getPrincipal(authHeaders), + expected.getOwner()); + + // Entity specific validation + TestUtils.validateTags(expected.getTags(), patched.getTags()); + TestUtils.validateEntityReference(expected.getFollowers()); + } + + @Override + public GlossaryEntityInterface getEntityInterface(Glossary entity) { + return new GlossaryEntityInterface(entity); + } + + @Override + public void validateGetWithDifferentFields(Glossary entity, boolean byName) throws HttpResponseException {} + + @Override + public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException { + if (expected == actual) { + return; + } + assertCommonFieldChange(fieldName, expected, actual); + } +} diff --git a/ingestion/examples/sample_data/thesauruses/hobbies-SKOS.json b/ingestion/examples/sample_data/thesauruses/hobbies-SKOS.json new file mode 100644 index 00000000000..8930ecbdf52 --- /dev/null +++ b/ingestion/examples/sample_data/thesauruses/hobbies-SKOS.json @@ -0,0 +1,1332 @@ +[ + { + "@id": "http://hobbies.com/achievement_hobbies", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Achievement hobbies", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#topConceptOf": [ + { + "@id": "http://hobbies.com/scheme" + } + ] + }, + { + "@id": "http://hobbies.com/air_sports", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Air sports", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/airsoft", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Airsoft", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/amateur_radio", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Amateur radio", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/antiquities", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Antiquities", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/archery", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Archery", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/audiophilia", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Audiophilia", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/auto_audiophilia", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Auto audiophilia", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/auto_racing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Auto racing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/badminton", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Badminton", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/baseball", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Baseball", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/basketball", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Basketball", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/baton_twirling", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Baton twirling", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/billiards", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Billiards", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/board_sports", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Board sports", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/bonsai", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Bonsai", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/bowling", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Bowling", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/boxing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Boxing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/chess", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Chess", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/climbing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Climbing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/coin_collecting", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Coin collecting", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/collection_hobbies", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Collection hobbies", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#topConceptOf": [ + { + "@id": "http://hobbies.com/scheme" + } + ] + }, + { + "@id": "http://hobbies.com/competition_hobbies", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Competition hobbies", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#topConceptOf": [ + { + "@id": "http://hobbies.com/scheme" + } + ] + }, + { + "@id": "http://hobbies.com/computer_programming", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Computer programming", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/cricket", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Cricket", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/cycling", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Cycling", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/dance", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Dance", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/darts", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Darts", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/disc_golf", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Disc golf", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/equestrianism", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Equestrianism", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/fencing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Fencing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/figure_skating", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Figure skating", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/fishing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Fishing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/fossil_collecting", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Fossil collecting", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/gardening", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Gardening", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/golfing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Golfing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/gymnastics", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Gymnastics", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/handball", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Handball", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/home_automation", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Home automation", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/ice_hockey", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Ice hockey", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/indoors", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Collection" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "indoors", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#member": [ + { + "@id": "http://hobbies.com/audiophilia" + }, + { + "@id": "http://hobbies.com/baton_twirling" + }, + { + "@id": "http://hobbies.com/bonsai" + }, + { + "@id": "http://hobbies.com/computer_programming" + }, + { + "@id": "http://hobbies.com/dance" + }, + { + "@id": "http://hobbies.com/amateur_radio" + }, + { + "@id": "http://hobbies.com/home_automation" + }, + { + "@id": "http://hobbies.com/knapping" + }, + { + "@id": "http://hobbies.com/lapidary" + }, + { + "@id": "http://hobbies.com/locksport" + }, + { + "@id": "http://hobbies.com/scrapbooking" + }, + { + "@id": "http://hobbies.com/coin_collecting" + }, + { + "@id": "http://hobbies.com/stamp_collecting" + }, + { + "@id": "http://hobbies.com/billiards" + }, + { + "@id": "http://hobbies.com/bowling" + }, + { + "@id": "http://hobbies.com/boxing" + }, + { + "@id": "http://hobbies.com/chess" + }, + { + "@id": "http://hobbies.com/darts" + }, + { + "@id": "http://hobbies.com/fencing" + }, + { + "@id": "http://hobbies.com/role-playing_games" + }, + { + "@id": "http://hobbies.com/table_football" + }, + { + "@id": "http://hobbies.com/handball" + }, + { + "@id": "http://hobbies.com/martial_arts" + }, + { + "@id": "http://hobbies.com/video_gaming" + } + ] + }, + { + "@id": "http://hobbies.com/kart_racing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Kart racing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/kayaking", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Kayaking", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/kitesurfing", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Kitesurfing", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/knapping", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Knapping", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/lapidary", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Lapidary", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/locksport", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Locksport", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/martial_arts", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Martial arts", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/metal_detecting", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Metal detecting", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/mineral_collecting", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Mineral collecting", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/model_building", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Model building", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/model_rocketry", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Model rocketry", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/model_building" + } + ] + }, + { + "@id": "http://hobbies.com/mountainbiking", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Mountainbiking", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/outdoors", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Collection" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "outdoors", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#member": [ + { + "@id": "http://hobbies.com/air_sports" + }, + { + "@id": "http://hobbies.com/board_sports" + }, + { + "@id": "http://hobbies.com/kayaking" + }, + { + "@id": "http://hobbies.com/kitesurfing" + }, + { + "@id": "http://hobbies.com/mountainbiking" + }, + { + "@id": "http://hobbies.com/scuba_diving" + }, + { + "@id": "http://hobbies.com/snowmobiling" + }, + { + "@id": "http://hobbies.com/antiquities" + }, + { + "@id": "http://hobbies.com/auto_audiophilia" + }, + { + "@id": "http://hobbies.com/fossil_collecting" + }, + { + "@id": "http://hobbies.com/metal_detecting" + }, + { + "@id": "http://hobbies.com/mineral_collecting" + }, + { + "@id": "http://hobbies.com/airsoft" + }, + { + "@id": "http://hobbies.com/archery" + }, + { + "@id": "http://hobbies.com/auto_racing" + }, + { + "@id": "http://hobbies.com/badminton" + }, + { + "@id": "http://hobbies.com/baseball" + }, + { + "@id": "http://hobbies.com/basketball" + }, + { + "@id": "http://hobbies.com/climbing" + }, + { + "@id": "http://hobbies.com/cricket" + }, + { + "@id": "http://hobbies.com/cycling" + }, + { + "@id": "http://hobbies.com/disc_golf" + }, + { + "@id": "http://hobbies.com/equestrianism" + }, + { + "@id": "http://hobbies.com/figure_skating" + }, + { + "@id": "http://hobbies.com/fishing" + }, + { + "@id": "http://hobbies.com/golfing" + }, + { + "@id": "http://hobbies.com/gymnastics" + }, + { + "@id": "http://hobbies.com/ice_hockey" + }, + { + "@id": "http://hobbies.com/kart_racing" + }, + { + "@id": "http://hobbies.com/paintball" + }, + { + "@id": "http://hobbies.com/rugby_league" + }, + { + "@id": "http://hobbies.com/swimming" + }, + { + "@id": "http://hobbies.com/tennis" + } + ] + }, + { + "@id": "http://hobbies.com/paintball", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Paintball", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/photography", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Photography", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/radio-controlled_modeling", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Radio-controlled modeling", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/model_building" + } + ] + }, + { + "@id": "http://hobbies.com/role-playing_games", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Role-playing games", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/rugby_league", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Rugby league", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/scale_modeling", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Scale modeling", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/model_building" + } + ] + }, + { + "@id": "http://hobbies.com/scheme" + }, + { + "@id": "http://hobbies.com/scrapbooking", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Scrapbooking", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/scuba_diving", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Scuba diving", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/snowmobiling", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Snowmobiling", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/achievement_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/stamp_collecting", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Stamp collecting", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/collection_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/swimming", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Swimming", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/table_football", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Table football", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/tennis", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Tennis", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://hobbies.com/video_gaming", + "@type": [ + "http://www.w3.org/2004/02/skos/core#Concept" + ], + "http://www.w3.org/2004/02/skos/core#prefLabel": [ + { + "@value": "Video gaming", + "@language": "en" + } + ], + "http://www.w3.org/2004/02/skos/core#broader": [ + { + "@id": "http://hobbies.com/competition_hobbies" + } + ] + }, + { + "@id": "http://www.w3.org/2004/02/skos/core#Collection" + }, + { + "@id": "http://www.w3.org/2004/02/skos/core#Concept" + } + ] \ No newline at end of file diff --git a/ingestion/pipelines/metadata_to_es.json b/ingestion/pipelines/metadata_to_es.json index a95361a838f..c459523c1a5 100644 --- a/ingestion/pipelines/metadata_to_es.json +++ b/ingestion/pipelines/metadata_to_es.json @@ -5,6 +5,7 @@ "include_tables": "true", "include_topics": "true", "include_dashboards": "true", + "include_glossary": "true", "limit_records": 10 } }, @@ -14,6 +15,7 @@ "index_tables": "true", "index_topics": "true", "index_dashboards": "true", + "index_glossary": "true", "es_host": "localhost", "es_port": 9200 } diff --git a/ingestion/src/metadata/ingestion/models/table_metadata.py b/ingestion/src/metadata/ingestion/models/table_metadata.py index 416c226355b..729a1dd2872 100644 --- a/ingestion/src/metadata/ingestion/models/table_metadata.py +++ b/ingestion/src/metadata/ingestion/models/table_metadata.py @@ -104,6 +104,23 @@ class TopicESDocument(BaseModel): doc_as_upsert: bool = True +class GlossaryESDocument(BaseModel): + """Glossary Elastic Search Mapping doc""" + + glossary_id: str + glossary_name: str + entity_type: str = "glossary" + suggest: List[dict] + description: Optional[str] = None + last_updated_timestamp: Optional[int] + tags: List[str] + fqdn: str + tier: Optional[str] = None + schema_description: Optional[str] = None + owner: str + followers: List[str] + + class DashboardESDocument(BaseModel): """Elastic Search Mapping doc for Dashboards""" diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index e7b92e49607..648131b93c7 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -31,6 +31,7 @@ from metadata.generated.schema.entity.data.mlmodel import MlModel from metadata.generated.schema.entity.data.pipeline import Pipeline from metadata.generated.schema.entity.data.report import Report from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.data.glossary import Glossary from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.policies.policy import Policy from metadata.generated.schema.entity.services.dashboardService import DashboardService @@ -162,6 +163,11 @@ class OpenMetadata( ): return "/mlmodels" + if issubclass( + entity, get_args(Union[Glossary, self.get_create_entity_type(Glossary)]) + ): + return "/glossary" + if issubclass( entity, get_args(Union[Chart, self.get_create_entity_type(Chart)]) ): diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch.py b/ingestion/src/metadata/ingestion/sink/elasticsearch.py index 13f31651948..674ef6bdac0 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch.py @@ -26,6 +26,7 @@ from metadata.generated.schema.entity.data.dashboard import Dashboard from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.pipeline import Pipeline, Task from metadata.generated.schema.entity.data.table import Column, Table +from metadata.generated.schema.entity.data.glossary import Glossary from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.services.dashboardService import DashboardService from metadata.generated.schema.entity.services.databaseService import DatabaseService @@ -39,6 +40,7 @@ from metadata.ingestion.models.table_metadata import ( DashboardESDocument, PipelineESDocument, TableESDocument, + GlossaryESDocument, TopicESDocument, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata @@ -47,6 +49,7 @@ from metadata.ingestion.sink.elasticsearch_constants import ( DASHBOARD_ELASTICSEARCH_INDEX_MAPPING, PIPELINE_ELASTICSEARCH_INDEX_MAPPING, TABLE_ELASTICSEARCH_INDEX_MAPPING, + GLOSSARY_ELASTICSEARCH_INDEX_MAPPING, TOPIC_ELASTICSEARCH_INDEX_MAPPING, ) @@ -64,10 +67,12 @@ class ElasticSearchConfig(ConfigModel): es_password: Optional[str] = None index_tables: Optional[bool] = True index_topics: Optional[bool] = True + index_glossary: Optional[bool] = True index_dashboards: Optional[bool] = True index_pipelines: Optional[bool] = True index_dbt_models: Optional[bool] = True table_index_name: str = "table_search_index" + glossary_index_name: str = "glossary_search_index" topic_index_name: str = "topic_search_index" dashboard_index_name: str = "dashboard_search_index" pipeline_index_name: str = "pipeline_search_index" @@ -131,6 +136,10 @@ class ElasticsearchSink(Sink[Entity]): self._check_or_create_index( self.config.table_index_name, TABLE_ELASTICSEARCH_INDEX_MAPPING ) + if self.config.index_glossary: + self._check_or_create_index( + self.config.glossary_index_name, GLOSSARY_ELASTICSEARCH_INDEX_MAPPING + ) if self.config.index_topics: self._check_or_create_index( self.config.topic_index_name, TOPIC_ELASTICSEARCH_INDEX_MAPPING @@ -183,6 +192,13 @@ class ElasticsearchSink(Sink[Entity]): body=table_doc.json(), request_timeout=self.config.timeout, ) + if isinstance(record, Glossary): + glossary_doc = self._create_glossary_es_doc(record) + self.elasticsearch_client.index( + index=self.config.glossary_index_name, + id=str(glossary_doc.glossary_id), + body=glossary_doc.json(), + ) if isinstance(record, Topic): topic_doc = self._create_topic_es_doc(record) self.elasticsearch_client.index( @@ -443,6 +459,61 @@ class ElasticsearchSink(Sink[Entity]): return pipeline_doc + def _create_glossary_es_doc(self, glossary: Glossary): + fqdn = glossary.fullyQualifiedName + suggest = [ + { + "input": [ + glossary.displayName if glossary.displayName else glossary.name + ], + "weight": 10, + } + ] + tags = set() + timestamp = time.time() + glossary_owner = ( + str(glossary.owner.id.__root__) if glossary.owner is not None else "" + ) + glossary_followers = [] + if glossary.followers: + for follower in glossary.followers.__root__: + glossary_followers.append(str(follower.id.__root__)) + tier = None + for glossary_tag in glossary.tags: + if "Tier" in glossary_tag.tagFQN: + tier = glossary_tag.tagFQN + else: + tags.add(glossary_tag.tagFQN) + # tasks: List[Task] = glossary.tasks # TODO Handle Glossary words + # task_names = [] + # task_descriptions = [] + # for task in tasks: + # task_names.append(task.displayName) + # if task.description is not None: + # task_descriptions.append(task.description) + # if tags in task and len(task.tags) > 0: + # for col_tag in task.tags: + # tags.add(col_tag.tagFQN) + + glossary_doc = GlossaryESDocument( + glossary_id=str(glossary.id.__root__), + glossary_name=glossary.displayName + if glossary.displayName + else glossary.name, + # task_names=task_names, # TODO Handle Glossary words + # task_descriptions=task_descriptions, + suggest=suggest, + description=glossary.description, + last_updated_timestamp=timestamp, + tier=tier, + tags=list(tags), + fqdn=fqdn, + owner=glossary_owner, + followers=glossary_followers, + ) + + return glossary_doc + def _get_charts(self, chart_refs: Optional[List[entityReference.EntityReference]]): charts = [] if chart_refs: diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py index b26489f708e..bf99eaeb592 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py @@ -200,6 +200,48 @@ TOPIC_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( """ ) +GLOSSARY_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( + """ + { + "mappings":{ + "properties": { + "glossary_name": { + "type":"text" + }, + "display_name": { + "type": "text" + }, + "owner": { + "type": "text" + }, + "followers": { + "type": "keyword" + }, + "last_updated_timestamp": { + "type": "date", + "format": "epoch_second" + }, + "description": { + "type": "text" + }, + "tier": { + "type": "keyword" + }, + "tags": { + "type": "keyword" + }, + "entity_type": { + "type": "keyword" + }, + "suggest": { + "type": "completion" + } + } + } + } + """ +) + DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( """ { diff --git a/ingestion/src/metadata/ingestion/source/metadata.py b/ingestion/src/metadata/ingestion/source/metadata.py index 4d6b3f1ffe2..8a00d332798 100644 --- a/ingestion/src/metadata/ingestion/source/metadata.py +++ b/ingestion/src/metadata/ingestion/source/metadata.py @@ -23,6 +23,7 @@ from metadata.ingestion.api.common import Entity, WorkflowContext from metadata.ingestion.api.source import Source, SourceStatus from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig +from metadata.ingestion.ometa.openmetadata_rest import Glossary logger = logging.getLogger(__name__) @@ -33,6 +34,7 @@ class MetadataTablesRestSourceConfig(ConfigModel): include_tables: Optional[bool] = True include_topics: Optional[bool] = True include_dashboards: Optional[bool] = True + include_glossary: Optional[bool] = True include_pipelines: Optional[bool] = True limit_records: int = 1000 @@ -60,6 +62,10 @@ class MetadataSourceStatus(SourceStatus): self.success.append(table_name) logger.info("Table Scanned: %s", table_name) + def scanned_glossary(self, glossary_name: str) -> None: + self.success.append(glossary_name) + logger.info("Glossary Scanned: {}".format(glossary_name)) + def scanned_topic(self, topic_name: str) -> None: """scanned topic method @@ -143,6 +149,7 @@ class MetadataSource(Source[Entity]): def next_record(self) -> Iterable[Entity]: yield from self.fetch_table() yield from self.fetch_topic() + yield from self.fetch_glossary() yield from self.fetch_dashboard() yield from self.fetch_pipeline() @@ -247,6 +254,23 @@ class MetadataSource(Source[Entity]): break after = pipeline_entities.after + def fetch_glossary(self) -> Glossary: + if self.config.include_glossary: + after = None + while True: + glossary_entities = self.metadata.list_entities( + entity=Glossary, + fields=["owner", "tags", "followers"], + after=after, + limit=self.config.limit_records, + ) + for glossary in glossary_entities.entities: + self.status.scanned_glossary(glossary.name) + yield glossary + if glossary_entities.after is None: + break + after = glossary_entities.after + def get_status(self) -> SourceStatus: return self.status diff --git a/ingestion/tests/unit/test_ometa_endpoints.py b/ingestion/tests/unit/test_ometa_endpoints.py index 856875d63d5..633c5ef26e4 100644 --- a/ingestion/tests/unit/test_ometa_endpoints.py +++ b/ingestion/tests/unit/test_ometa_endpoints.py @@ -27,6 +27,7 @@ from metadata.generated.schema.entity.data.mlmodel import MlModel from metadata.generated.schema.entity.data.pipeline import Pipeline from metadata.generated.schema.entity.data.report import Report from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.data.glossary import Glossary from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.services.dashboardService import DashboardService from metadata.generated.schema.entity.services.databaseService import DatabaseService @@ -51,6 +52,10 @@ class OMetaEndpointTest(TestCase): Pass Entities and test their suffix generation """ + + # Glossary + self.assertEqual(self.metadata.get_suffix(Glossary), "/glossary") + # ML self.assertEqual(self.metadata.get_suffix(MlModel), "/mlmodels")