yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-01 02:56:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for versioning APIs and backend storage

Browse Source
This commit is contained in:
sureshms 2021-10-25 10:37:29 -07:00
parent 1633c21675
commit 71df299dd1
4 changed files with 117 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java
View File

@ -47,11 +47,14 @@ import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.UsageDetails;
import org.openmetadata.catalog.type.UsageStats;
import org.openmetadata.catalog.util.EntityUtil;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
public interface CollectionDAO {
@CreateSqlObject
@ -197,6 +200,41 @@ public interface CollectionDAO {
@SqlQuery("SELECT json FROM entity_extension WHERE id = :id AND extension = :extension")
String getExtension(@Bind("id") String id, @Bind("extension") String extension);
@RegisterRowMapper(EntityVersionMapper.class)
@SqlQuery("SELECT extension, json FROM entity_extension WHERE id = :id AND extension " +
"LIKE CONCAT (:extensionPrefix, '.%')")
List<EntityVersionPair> getEntityVersions(@Bind("id") String id, @Bind("extension") String extensionPrefix);
@RegisterRowMapper(EntityVersionMapper.class)
@SqlQuery("SELECT json FROM entity_extension WHERE id = :id AND extension = :extension")
String getEntityVersion(@Bind("id") String id, @Bind("extension") String extension);
}
class EntityVersionPair {
private Double version;
private String entityJson;
public Double getVersion() {
return version;
}
public String getEntityJson() {
return entityJson;
}
public EntityVersionPair(Double version, String json) {
this.version = version;
this.entityJson = json;
}
}
class EntityVersionMapper implements RowMapper<EntityVersionPair> {
@Override
public EntityVersionPair map(ResultSet rs, StatementContext ctx) throws SQLException {
Double version = EntityUtil.getVersion(rs.getString("extension"));
return new EntityVersionPair(version, rs.getString("json"));
}
}
interface EntityRelationshipDAO {

40
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java
View File

@ -2,6 +2,7 @@ package org.openmetadata.catalog.jdbi3;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.jdbi3.CollectionDAO.EntityVersionPair;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityHistory;
import org.openmetadata.catalog.type.EntityReference;
@ -22,6 +23,7 @@ import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.Locale;
@ -118,6 +120,26 @@ public abstract class EntityRepository<T> {
return getResultList(entities, beforeCursor, afterCursor, total);
}
@Transaction
public T getVersion(String id, String version) throws IOException {
String extension = EntityUtil.getVersionExtension(entityName, Double.valueOf(version));
String json = daoCollection.entityExtensionDAO().getEntityVersion(id, extension);
return JsonUtils.readValue(json, entityClass);
}
@Transaction
public EntityHistory listVersions(String id) throws IOException, ParseException {
T latest = setFields(dao.findEntityById(UUID.fromString(id)), putFields);
String extensionPrefix = EntityUtil.getVersionExtensionPrefix(entityName);
List<EntityVersionPair> oldVersions = daoCollection.entityExtensionDAO().getEntityVersions(id, extensionPrefix);
oldVersions.sort(Comparator.comparing(EntityVersionPair::getVersion).reversed());
final List<Object> allVersions = new ArrayList<>();
allVersions.add(JsonUtils.pojoToJson(latest));
oldVersions.forEach(version -> allVersions.add(version.getEntityJson()));
return new EntityHistory().withEntityType(entityName).withVersions(allVersions);
}
@Transaction
public final T create(T entity) throws IOException, ParseException {
validate(entity);
@ -172,6 +194,12 @@ public abstract class EntityRepository<T> {
return entity;
}
/**
* Class that provides functionality related to entity versioning
*/
public static class EntityVersionHelper {
}
/**
* Class that performs PUT and PATCH UPDATE operation. Override {@code entitySpecificUpdate()} to add
* additional entity specific fields to be updated.
@ -284,15 +312,21 @@ public abstract class EntityRepository<T> {
return false;
}
public final void store() throws IOException {
private void storeOldVersion() throws IOException {
// TODO move this into a single palce
String extensionName = EntityUtil.getVersionExtension(entityName, original.getVersion());
daoCollection.entityExtensionDAO().insert(original.getId().toString(), extensionName, entityName,
JsonUtils.pojoToJson(original.getEntity()));
}
public final void store() throws IOException, ParseException {
if (updateVersion(original.getVersion())) {
// Store the old version
List<Object> versions = new ArrayList<>();
versions.add(original.getEntity());
EntityHistory history = new EntityHistory().withEntityType(entityName).withVersions(versions);
System.out.println(JsonUtils.pojoToJson(history, true));
storeOldVersion();
}
// TODO clean up entity name
EntityRepository.this.store(updated.getEntity(), true);
}
}

14
catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java
View File

@ -447,4 +447,18 @@ public final class EntityUtil {
return refList.stream().sorted(Comparator.comparing(EntityReference::getId)).map(EntityReference::getId)
.collect(Collectors.toList());
}
public static String getVersionExtension(String entityName, Double version) {
return String.format("%s.%s.%s", entityName, "version", version.toString());
}
public static String getVersionExtensionPrefix(String entityName) {
return String.format("%s.%s", entityName, "version");
}
public static Double getVersion(String extension) {
String[] s = extension.split("\\.");
String versionString = s[2] + "." + s[3];
return Double.valueOf(versionString);
}
}

29
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java
View File

@ -36,6 +36,7 @@ import org.openmetadata.catalog.entity.services.DatabaseService;
import org.openmetadata.catalog.entity.teams.Team;
import org.openmetadata.catalog.entity.teams.User;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.resources.EntityTestHelper;
import org.openmetadata.catalog.resources.databases.TableResource.TableList;
import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest;
import org.openmetadata.catalog.resources.tags.TagResourceTest;
@ -55,6 +56,7 @@ import org.openmetadata.catalog.type.TableJoins;
import org.openmetadata.catalog.type.TableProfile;
import org.openmetadata.catalog.type.TableType;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.RestUtil;
@ -113,7 +115,7 @@ import static org.openmetadata.catalog.util.TestUtils.userAuthHeaders;
import static org.openmetadata.common.utils.CommonUtil.getDateStringByOffset;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class TableResourceTest extends CatalogApplicationTest {
public class TableResourceTest extends EntityTestHelper<Table> {
private static final Logger LOG = LoggerFactory.getLogger(TableResourceTest.class);
public static Database DATABASE;
public static final TagLabel USER_ADDRESS_TAG_LABEL = new TagLabel().withTagFQN("User.Address");
@ -1526,4 +1528,29 @@ public class TableResourceTest extends CatalogApplicationTest {
assertEquals(tableProfile, storedProfile);
}
}
@Override
public Table createEntity(Object createRequest, Map<String, String> authHeaders) {
return null;
}
@Override
public Table getEntity(UUID id, Map<String, String> authHeaders) {
return null;
}
@Override
public Table validateEntity(Table entity, Object createRequest) {
return null;
}
@Override
public EntityInterface<Table> getEntityInterface(Table entity) {
return null;
}
@Override
public Table validateCommonFields(Table entity) {
return null;
}
}