Add support for versioning APIs and backend storage

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java

@@ -47,11 +47,14 @@ import org.openmetadata.catalog.type.EntityReference;
 import org.openmetadata.catalog.type.TagLabel;
 import org.openmetadata.catalog.type.UsageDetails;
 import org.openmetadata.catalog.type.UsageStats;
+import org.openmetadata.catalog.util.EntityUtil;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
 
 public interface CollectionDAO {
   @CreateSqlObject
@@ -197,6 +200,41 @@ public interface CollectionDAO {
 
     @SqlQuery("SELECT json FROM entity_extension WHERE id = :id AND extension = :extension")
     String getExtension(@Bind("id") String id, @Bind("extension") String extension);
+
+    @RegisterRowMapper(EntityVersionMapper.class)
+    @SqlQuery("SELECT extension, json FROM entity_extension WHERE id = :id AND extension " +
+            "LIKE CONCAT (:extensionPrefix, '.%')")
+    List<EntityVersionPair> getEntityVersions(@Bind("id") String id, @Bind("extension") String extensionPrefix);
+
+    @RegisterRowMapper(EntityVersionMapper.class)
+    @SqlQuery("SELECT json FROM entity_extension WHERE id = :id AND extension = :extension")
+    String getEntityVersion(@Bind("id") String id, @Bind("extension") String extension);
+  }
+
+  class EntityVersionPair {
+    private Double version;
+    private String entityJson;
+
+    public Double getVersion() {
+      return version;
+    }
+
+    public String getEntityJson() {
+      return entityJson;
+    }
+
+    public EntityVersionPair(Double version, String json) {
+      this.version = version;
+      this.entityJson = json;
+    }
+  }
+
+  class EntityVersionMapper implements RowMapper<EntityVersionPair> {
+    @Override
+    public EntityVersionPair map(ResultSet rs, StatementContext ctx) throws SQLException {
+      Double version = EntityUtil.getVersion(rs.getString("extension"));
+      return new EntityVersionPair(version, rs.getString("json"));
+    }
   }
 
   interface EntityRelationshipDAO {

catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java

@@ -2,6 +2,7 @@ package org.openmetadata.catalog.jdbi3;
 
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.catalog.Entity;
+import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityVersionPair;
 import org.openmetadata.catalog.type.ChangeDescription;
 import org.openmetadata.catalog.type.EntityHistory;
 import org.openmetadata.catalog.type.EntityReference;
@@ -22,6 +23,7 @@ import java.io.UnsupportedEncodingException;
 import java.security.GeneralSecurityException;
 import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
@@ -118,6 +120,26 @@ public abstract class EntityRepository<T> {
     return getResultList(entities, beforeCursor, afterCursor, total);
   }
 
+  @Transaction
+  public T getVersion(String id, String version) throws IOException {
+    String extension = EntityUtil.getVersionExtension(entityName, Double.valueOf(version));
+    String json = daoCollection.entityExtensionDAO().getEntityVersion(id, extension);
+    return JsonUtils.readValue(json, entityClass);
+  }
+
+  @Transaction
+  public EntityHistory listVersions(String id) throws IOException, ParseException {
+    T latest = setFields(dao.findEntityById(UUID.fromString(id)), putFields);
+    String extensionPrefix = EntityUtil.getVersionExtensionPrefix(entityName);
+    List<EntityVersionPair> oldVersions = daoCollection.entityExtensionDAO().getEntityVersions(id, extensionPrefix);
+    oldVersions.sort(Comparator.comparing(EntityVersionPair::getVersion).reversed());
+
+    final List<Object> allVersions = new ArrayList<>();
+    allVersions.add(JsonUtils.pojoToJson(latest));
+    oldVersions.forEach(version -> allVersions.add(version.getEntityJson()));
+    return new EntityHistory().withEntityType(entityName).withVersions(allVersions);
+  }
+
   @Transaction
   public final T create(T entity) throws IOException, ParseException {
     validate(entity);
@@ -172,6 +194,12 @@ public abstract class EntityRepository<T> {
     return entity;
   }
 
+  /**
+   * Class that provides functionality related to entity versioning
+   */
+  public static class EntityVersionHelper {
+  }
+
   /**
    * Class that performs PUT and PATCH UPDATE operation. Override {@code entitySpecificUpdate()} to add
    * additional entity specific fields to be updated.
@@ -284,15 +312,21 @@ public abstract class EntityRepository<T> {
       return false;
     }
 
-    public final void store() throws IOException {
+    private void storeOldVersion() throws IOException {
+      // TODO move this into a single palce
+      String extensionName = EntityUtil.getVersionExtension(entityName, original.getVersion());
+      daoCollection.entityExtensionDAO().insert(original.getId().toString(), extensionName, entityName,
+              JsonUtils.pojoToJson(original.getEntity()));
+    }
+
+    public final void store() throws IOException, ParseException {
       if (updateVersion(original.getVersion())) {
         // Store the old version
         List<Object> versions = new ArrayList<>();
         versions.add(original.getEntity());
         EntityHistory history = new EntityHistory().withEntityType(entityName).withVersions(versions);
-        System.out.println(JsonUtils.pojoToJson(history, true));
+        storeOldVersion();
       }
-      // TODO clean up entity name
       EntityRepository.this.store(updated.getEntity(), true);
     }
   }

catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java

@@ -447,4 +447,18 @@ public final class EntityUtil {
     return refList.stream().sorted(Comparator.comparing(EntityReference::getId)).map(EntityReference::getId)
             .collect(Collectors.toList());
   }
+
+  public static String getVersionExtension(String entityName, Double version) {
+    return String.format("%s.%s.%s", entityName, "version", version.toString());
+  }
+
+  public static String getVersionExtensionPrefix(String entityName) {
+    return String.format("%s.%s", entityName, "version");
+  }
+
+  public static Double getVersion(String extension) {
+    String[] s = extension.split("\\.");
+    String versionString = s[2] + "." + s[3];
+    return Double.valueOf(versionString);
+  }
 }

catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java

@@ -36,6 +36,7 @@ import org.openmetadata.catalog.entity.services.DatabaseService;
 import org.openmetadata.catalog.entity.teams.Team;
 import org.openmetadata.catalog.entity.teams.User;
 import org.openmetadata.catalog.exception.CatalogExceptionMessage;
+import org.openmetadata.catalog.resources.EntityTestHelper;
 import org.openmetadata.catalog.resources.databases.TableResource.TableList;
 import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest;
 import org.openmetadata.catalog.resources.tags.TagResourceTest;
@@ -55,6 +56,7 @@ import org.openmetadata.catalog.type.TableJoins;
 import org.openmetadata.catalog.type.TableProfile;
 import org.openmetadata.catalog.type.TableType;
 import org.openmetadata.catalog.type.TagLabel;
+import org.openmetadata.catalog.util.EntityInterface;
 import org.openmetadata.catalog.util.EntityUtil.Fields;
 import org.openmetadata.catalog.util.JsonUtils;
 import org.openmetadata.catalog.util.RestUtil;
@@ -113,7 +115,7 @@ import static org.openmetadata.catalog.util.TestUtils.userAuthHeaders;
 import static org.openmetadata.common.utils.CommonUtil.getDateStringByOffset;
 
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
-public class TableResourceTest extends CatalogApplicationTest {
+public class TableResourceTest extends EntityTestHelper<Table> {
   private static final Logger LOG = LoggerFactory.getLogger(TableResourceTest.class);
   public static Database DATABASE;
   public static final TagLabel USER_ADDRESS_TAG_LABEL = new TagLabel().withTagFQN("User.Address");
@@ -1526,4 +1528,29 @@ public class TableResourceTest extends CatalogApplicationTest {
       assertEquals(tableProfile, storedProfile);
     }
   }
+
+  @Override
+  public Table createEntity(Object createRequest, Map<String, String> authHeaders) {
+    return null;
+  }
+
+  @Override
+  public Table getEntity(UUID id, Map<String, String> authHeaders) {
+    return null;
+  }
+
+  @Override
+  public Table validateEntity(Table entity, Object createRequest) {
+    return null;
+  }
+
+  @Override
+  public EntityInterface<Table> getEntityInterface(Table entity) {
+    return null;
+  }
+
+  @Override
+  public Table validateCommonFields(Table entity) {
+    return null;
+  }
 }