From 71df299dd147f6fecbdaf7d780d360b230814395 Mon Sep 17 00:00:00 2001 From: sureshms Date: Mon, 25 Oct 2021 10:37:29 -0700 Subject: [PATCH] Add support for versioning APIs and backend storage --- .../catalog/jdbi3/CollectionDAO.java | 38 ++++++++++++++++++ .../catalog/jdbi3/EntityRepository.java | 40 +++++++++++++++++-- .../openmetadata/catalog/util/EntityUtil.java | 14 +++++++ .../databases/TableResourceTest.java | 29 +++++++++++++- 4 files changed, 117 insertions(+), 4 deletions(-) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java index f3d7693a3b1..6eedf40c37a 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java @@ -47,11 +47,14 @@ import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.type.UsageDetails; import org.openmetadata.catalog.type.UsageStats; +import org.openmetadata.catalog.util.EntityUtil; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; public interface CollectionDAO { @CreateSqlObject @@ -197,6 +200,41 @@ public interface CollectionDAO { @SqlQuery("SELECT json FROM entity_extension WHERE id = :id AND extension = :extension") String getExtension(@Bind("id") String id, @Bind("extension") String extension); + + @RegisterRowMapper(EntityVersionMapper.class) + @SqlQuery("SELECT extension, json FROM entity_extension WHERE id = :id AND extension " + + "LIKE CONCAT (:extensionPrefix, '.%')") + List getEntityVersions(@Bind("id") String id, @Bind("extension") String extensionPrefix); + + @RegisterRowMapper(EntityVersionMapper.class) + @SqlQuery("SELECT json FROM entity_extension WHERE id = :id AND extension = :extension") + String getEntityVersion(@Bind("id") String id, @Bind("extension") String extension); + } + + class EntityVersionPair { + private Double version; + private String entityJson; + + public Double getVersion() { + return version; + } + + public String getEntityJson() { + return entityJson; + } + + public EntityVersionPair(Double version, String json) { + this.version = version; + this.entityJson = json; + } + } + + class EntityVersionMapper implements RowMapper { + @Override + public EntityVersionPair map(ResultSet rs, StatementContext ctx) throws SQLException { + Double version = EntityUtil.getVersion(rs.getString("extension")); + return new EntityVersionPair(version, rs.getString("json")); + } } interface EntityRelationshipDAO { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java index 2a1a5bca6c3..ae811597cb8 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java @@ -2,6 +2,7 @@ package org.openmetadata.catalog.jdbi3; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityVersionPair; import org.openmetadata.catalog.type.ChangeDescription; import org.openmetadata.catalog.type.EntityHistory; import org.openmetadata.catalog.type.EntityReference; @@ -22,6 +23,7 @@ import java.io.UnsupportedEncodingException; import java.security.GeneralSecurityException; import java.text.ParseException; import java.util.ArrayList; +import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.Locale; @@ -118,6 +120,26 @@ public abstract class EntityRepository { return getResultList(entities, beforeCursor, afterCursor, total); } + @Transaction + public T getVersion(String id, String version) throws IOException { + String extension = EntityUtil.getVersionExtension(entityName, Double.valueOf(version)); + String json = daoCollection.entityExtensionDAO().getEntityVersion(id, extension); + return JsonUtils.readValue(json, entityClass); + } + + @Transaction + public EntityHistory listVersions(String id) throws IOException, ParseException { + T latest = setFields(dao.findEntityById(UUID.fromString(id)), putFields); + String extensionPrefix = EntityUtil.getVersionExtensionPrefix(entityName); + List oldVersions = daoCollection.entityExtensionDAO().getEntityVersions(id, extensionPrefix); + oldVersions.sort(Comparator.comparing(EntityVersionPair::getVersion).reversed()); + + final List allVersions = new ArrayList<>(); + allVersions.add(JsonUtils.pojoToJson(latest)); + oldVersions.forEach(version -> allVersions.add(version.getEntityJson())); + return new EntityHistory().withEntityType(entityName).withVersions(allVersions); + } + @Transaction public final T create(T entity) throws IOException, ParseException { validate(entity); @@ -172,6 +194,12 @@ public abstract class EntityRepository { return entity; } + /** + * Class that provides functionality related to entity versioning + */ + public static class EntityVersionHelper { + } + /** * Class that performs PUT and PATCH UPDATE operation. Override {@code entitySpecificUpdate()} to add * additional entity specific fields to be updated. @@ -284,15 +312,21 @@ public abstract class EntityRepository { return false; } - public final void store() throws IOException { + private void storeOldVersion() throws IOException { + // TODO move this into a single palce + String extensionName = EntityUtil.getVersionExtension(entityName, original.getVersion()); + daoCollection.entityExtensionDAO().insert(original.getId().toString(), extensionName, entityName, + JsonUtils.pojoToJson(original.getEntity())); + } + + public final void store() throws IOException, ParseException { if (updateVersion(original.getVersion())) { // Store the old version List versions = new ArrayList<>(); versions.add(original.getEntity()); EntityHistory history = new EntityHistory().withEntityType(entityName).withVersions(versions); - System.out.println(JsonUtils.pojoToJson(history, true)); + storeOldVersion(); } - // TODO clean up entity name EntityRepository.this.store(updated.getEntity(), true); } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java index d552d2c926f..8a996dc24f7 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java @@ -447,4 +447,18 @@ public final class EntityUtil { return refList.stream().sorted(Comparator.comparing(EntityReference::getId)).map(EntityReference::getId) .collect(Collectors.toList()); } + + public static String getVersionExtension(String entityName, Double version) { + return String.format("%s.%s.%s", entityName, "version", version.toString()); + } + + public static String getVersionExtensionPrefix(String entityName) { + return String.format("%s.%s", entityName, "version"); + } + + public static Double getVersion(String extension) { + String[] s = extension.split("\\."); + String versionString = s[2] + "." + s[3]; + return Double.valueOf(versionString); + } } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java index 08f5c2fa451..29a9ca05557 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java @@ -36,6 +36,7 @@ import org.openmetadata.catalog.entity.services.DatabaseService; import org.openmetadata.catalog.entity.teams.Team; import org.openmetadata.catalog.entity.teams.User; import org.openmetadata.catalog.exception.CatalogExceptionMessage; +import org.openmetadata.catalog.resources.EntityTestHelper; import org.openmetadata.catalog.resources.databases.TableResource.TableList; import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest; import org.openmetadata.catalog.resources.tags.TagResourceTest; @@ -55,6 +56,7 @@ import org.openmetadata.catalog.type.TableJoins; import org.openmetadata.catalog.type.TableProfile; import org.openmetadata.catalog.type.TableType; import org.openmetadata.catalog.type.TagLabel; +import org.openmetadata.catalog.util.EntityInterface; import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.JsonUtils; import org.openmetadata.catalog.util.RestUtil; @@ -113,7 +115,7 @@ import static org.openmetadata.catalog.util.TestUtils.userAuthHeaders; import static org.openmetadata.common.utils.CommonUtil.getDateStringByOffset; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class TableResourceTest extends CatalogApplicationTest { +public class TableResourceTest extends EntityTestHelper { private static final Logger LOG = LoggerFactory.getLogger(TableResourceTest.class); public static Database DATABASE; public static final TagLabel USER_ADDRESS_TAG_LABEL = new TagLabel().withTagFQN("User.Address"); @@ -1526,4 +1528,29 @@ public class TableResourceTest extends CatalogApplicationTest { assertEquals(tableProfile, storedProfile); } } + + @Override + public Table createEntity(Object createRequest, Map authHeaders) { + return null; + } + + @Override + public Table getEntity(UUID id, Map authHeaders) { + return null; + } + + @Override + public Table validateEntity(Table entity, Object createRequest) { + return null; + } + + @Override + public EntityInterface
getEntityInterface(Table entity) { + return null; + } + + @Override + public Table validateCommonFields(Table entity) { + return null; + } }