yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-01 13:13:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #7642 Entity versioning for Custom Properties change (#7678)

Browse Source
This commit is contained in:
Suresh Srinivas 2022-09-22 21:34:40 -07:00 committed by GitHub
parent 77b1cc5277
commit ff8a41febc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 126 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ChartRepository.java
View File

@ -14,8 +14,6 @@
package org.openmetadata.service.jdbi3;
import static org.openmetadata.service.Entity.FIELD_FOLLOWERS;
import static org.openmetadata.service.Entity.FIELD_OWNER;
import static org.openmetadata.service.Entity.FIELD_TAGS;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
@ -91,9 +89,7 @@ public class ChartRepository extends EntityRepository<Chart> {
@Override
public Chart setFields(Chart chart, Fields fields) throws IOException {
chart.setService(getContainer(chart.getId()));
chart.setOwner(fields.contains(FIELD_OWNER) ? getOwner(chart) : null);
chart.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(chart) : null);
chart.setTags(fields.contains(FIELD_TAGS) ? getTags(chart.getFullyQualifiedName()) : null);
return chart;
}

6
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardRepository.java
View File

@ -14,10 +14,7 @@
package org.openmetadata.service.jdbi3;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.service.Entity.FIELD_EXTENSION;
import static org.openmetadata.service.Entity.FIELD_FOLLOWERS;
import static org.openmetadata.service.Entity.FIELD_OWNER;
import static org.openmetadata.service.Entity.FIELD_TAGS;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
@ -61,15 +58,12 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
@Override
public Dashboard setFields(Dashboard dashboard, Fields fields) throws IOException {
dashboard.setService(getContainer(dashboard.getId()));
dashboard.setOwner(fields.contains(FIELD_OWNER) ? getOwner(dashboard) : null);
dashboard.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(dashboard) : null);
dashboard.setCharts(fields.contains("charts") ? getCharts(dashboard) : null);
dashboard.setTags(fields.contains(FIELD_TAGS) ? getTags(dashboard.getFullyQualifiedName()) : null);
dashboard.setUsageSummary(
fields.contains("usageSummary")
? EntityUtil.getLatestUsage(daoCollection.usageDAO(), dashboard.getId())
: null);
dashboard.setExtension(fields.contains(FIELD_EXTENSION) ? getExtension(dashboard) : null);
return dashboard;
}

83
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
View File

@ -29,6 +29,7 @@ import static org.openmetadata.service.util.EntityUtil.compareTagLabel;
import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch;
import static org.openmetadata.service.util.EntityUtil.fieldAdded;
import static org.openmetadata.service.util.EntityUtil.fieldDeleted;
import static org.openmetadata.service.util.EntityUtil.getExtensionField;
import static org.openmetadata.service.util.EntityUtil.nextMajorVersion;
import static org.openmetadata.service.util.EntityUtil.nextVersion;
import static org.openmetadata.service.util.EntityUtil.objectMatch;
@ -36,7 +37,6 @@ import static org.openmetadata.service.util.EntityUtil.tagLabelMatch;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.networknt.schema.JsonSchema;
import com.networknt.schema.ValidationMessage;
@ -283,7 +283,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
@Transaction
public final T get(UriInfo uriInfo, UUID id, Fields fields, Include include) throws IOException {
return withHref(uriInfo, setFields(dao.findEntityById(id, include), fields));
return withHref(uriInfo, setFieldsInternal(dao.findEntityById(id, include), fields));
}
@Transaction
@ -293,7 +293,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
@Transaction
public final T getByName(UriInfo uriInfo, String fqn, Fields fields, Include include) throws IOException {
return withHref(uriInfo, setFields(dao.findEntityByName(fqn, include), fields));
return withHref(uriInfo, setFieldsInternal(dao.findEntityByName(fqn, include), fields));
}
@Transaction
@ -306,7 +306,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
List<String> jsons = dao.listAfter(filter, limitParam + 1, after == null ? "" : RestUtil.decodeCursor(after));
for (String json : jsons) {
T entity = withHref(uriInfo, setFields(JsonUtils.readValue(json, entityClass), fields));
T entity = withHref(uriInfo, setFieldsInternal(JsonUtils.readValue(json, entityClass), fields));
entities.add(entity);
}
@ -332,7 +332,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
List<T> entities = new ArrayList<>();
for (String json : jsons) {
T entity = withHref(uriInfo, setFields(JsonUtils.readValue(json, entityClass), fields));
T entity = withHref(uriInfo, setFieldsInternal(JsonUtils.readValue(json, entityClass), fields));
entities.add(entity);
}
int total = dao.listCount(filter);
@ -358,7 +358,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
return JsonUtils.readValue(json, entityClass);
}
// If requested the latest version, return it from current version of the entity
T entity = setFields(dao.findEntityById(id, ALL), putFields);
T entity = setFieldsInternal(dao.findEntityById(id, ALL), putFields);
if (entity.getVersion().equals(requestedVersion)) {
return entity;
}
@ -368,7 +368,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
@Transaction
public EntityHistory listVersions(UUID id) throws IOException {
T latest = setFields(dao.findEntityById(id, ALL), putFields);
T latest = setFieldsInternal(dao.findEntityById(id, ALL), putFields);
String extensionPrefix = EntityUtil.getVersionExtensionPrefix(entityType);
List<ExtensionRecord> records = daoCollection.entityExtensionDAO().getExtensions(id.toString(), extensionPrefix);
List<EntityVersionPair> oldVersions = new ArrayList<>();
@ -389,15 +389,26 @@ public abstract class EntityRepository<T extends EntityInterface> {
@Transaction
public final T createInternal(T entity) throws IOException {
prepareInternal(entity);
return createNewEntity(entity);
}
private void prepareInternal(T entity) throws IOException {
prepare(entity);
validateExtension(entity);
return createNewEntity(entity);
}
T setFieldsInternal(T entity, Fields fields) throws IOException {
entity.setOwner(fields.contains(FIELD_OWNER) ? getOwner(entity) : null);
entity.setTags(fields.contains(FIELD_TAGS) ? getTags(entity.getFullyQualifiedName()) : null);
entity.setExtension(fields.contains("extension") ? getExtension(entity) : null);
setFields(entity, fields);
return entity;
}
@Transaction
public final PutResponse<T> createOrUpdate(UriInfo uriInfo, T original, T updated) throws IOException {
prepare(updated);
validateExtension(updated);
prepareInternal(updated);
// Check if there is any original, deleted or not
original = JsonUtils.readValue(dao.findJsonByFqn(original.getFullyQualifiedName(), ALL), entityClass);
if (original == null) {
@ -418,7 +429,6 @@ public abstract class EntityRepository<T extends EntityInterface> {
@Transaction
public final PutResponse<T> createOrUpdateInternal(UriInfo uriInfo, T updated) throws IOException {
validateExtension(updated);
// Check if there is any original, deleted or not
T original = JsonUtils.readValue(dao.findJsonByFqn(updated.getFullyQualifiedName(), ALL), entityClass);
if (original == null) {
@ -440,7 +450,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
@Transaction
public PutResponse<T> update(UriInfo uriInfo, T original, T updated) throws IOException {
// Get all the fields in the original entity that can be updated during PUT operation
setFields(original, putFields);
setFieldsInternal(original, putFields);
// If the entity state is soft-deleted, recursively undelete the entity and it's children
if (Boolean.TRUE.equals(original.getDeleted())) {
@ -457,16 +467,15 @@ public abstract class EntityRepository<T extends EntityInterface> {
@Transaction
public final PatchResponse<T> patch(UriInfo uriInfo, UUID id, String user, JsonPatch patch) throws IOException {
// Get all the fields in the original entity that can be updated during PATCH operation
T original = setFields(dao.findEntityById(id), patchFields);
T original = setFieldsInternal(dao.findEntityById(id), patchFields);
// Apply JSON patch to the original entity to get the updated entity
T updated = JsonUtils.applyPatch(original, patch, entityClass);
updated.setUpdatedBy(user);
updated.setUpdatedAt(System.currentTimeMillis());
prepare(updated);
prepareInternal(updated);
populateOwner(updated.getOwner());
validateExtension(updated);
restorePatchAttributes(original, updated);
// Update the attributes and relationships of an entity
@ -537,13 +546,13 @@ public abstract class EntityRepository<T extends EntityInterface> {
throws IOException {
T original = JsonUtils.readValue(json, entityClass);
preDelete(original);
setFields(original, putFields);
setFieldsInternal(original, putFields);
deleteChildren(id, recursive, hardDelete, updatedBy);
String changeType;
T updated = JsonUtils.readValue(json, entityClass);
setFields(updated, putFields); // we need service, database, databaseSchema to delete properly from ES.
setFieldsInternal(updated, putFields); // we need service, database, databaseSchema to delete properly from ES.
if (supportsSoftDelete && !hardDelete) {
updated.setUpdatedBy(updatedBy);
updated.setUpdatedAt(System.currentTimeMillis());
@ -739,11 +748,10 @@ public abstract class EntityRepository<T extends EntityInterface> {
if (records.isEmpty()) {
return null;
}
ObjectMapper mapper = new ObjectMapper();
ObjectNode objectNode = mapper.createObjectNode();
ObjectNode objectNode = JsonUtils.getObjectNode();
for (ExtensionRecord record : records) {
String fieldName = TypeRegistry.getPropertyName(record.getExtensionName());
objectNode.set(fieldName, mapper.readTree(record.getExtensionJson()));
objectNode.set(fieldName, JsonUtils.readTree(record.getExtensionJson()));
}
return objectNode;
}
@ -1239,12 +1247,47 @@ public abstract class EntityRepository<T extends EntityInterface> {
}
private void updateExtension() throws JsonProcessingException {
if (original.getExtension() == updated.getExtension()) {
return;
}
if (updatedByBot()) {
// Revert changes to extension field, if being updated by a bot
updated.setExtension(original.getExtension());
return;
}
List<JsonNode> added = new ArrayList<>();
List<JsonNode> deleted = new ArrayList<>();
JsonNode origFields = JsonUtils.valueToTree(original.getExtension());
JsonNode updatedFields = JsonUtils.valueToTree(updated.getExtension());
// Check for updated and deleted fields
for (Iterator<Entry<String, JsonNode>> it = origFields.fields(); it.hasNext(); ) {
Entry<String, JsonNode> orig = it.next();
JsonNode updated = updatedFields.get(orig.getKey());
if (updated == null) {
deleted.add(JsonUtils.getObjectNode(orig.getKey(), orig.getValue()));
} else {
// TODO converting to a string is a hack for now because JsonNode equals issues
recordChange(getExtensionField(orig.getKey()), orig.getValue().toString(), updated.toString());
}
}
// Check for added fields
for (Iterator<Entry<String, JsonNode>> it = updatedFields.fields(); it.hasNext(); ) {
Entry<String, JsonNode> updated = it.next();
JsonNode orig = origFields.get(updated.getKey());
if (orig == null) {
added.add(JsonUtils.getObjectNode(updated.getKey(), updated.getValue()));
}
}
if (!added.isEmpty()) {
fieldAdded(changeDescription, "extension", JsonUtils.pojoToJson(added));
}
if (!deleted.isEmpty()) {
fieldDeleted(changeDescription, "extension", JsonUtils.pojoToJson(deleted));
}
removeExtension(original);
storeExtension(updated);
}

4
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java
View File

@ -17,8 +17,6 @@
package org.openmetadata.service.jdbi3;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.service.Entity.FIELD_OWNER;
import static org.openmetadata.service.Entity.FIELD_TAGS;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
@ -51,8 +49,6 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
@Override
public Glossary setFields(Glossary glossary, Fields fields) throws IOException {
glossary.setOwner(fields.contains(FIELD_OWNER) ? getOwner(glossary) : null);
glossary.setTags(fields.contains(FIELD_TAGS) ? getTags(glossary.getName()) : null);
glossary.setReviewers(fields.contains("reviewers") ? getReviewers(glossary) : null);
return glossary.withUsageCount(fields.contains("usageCount") ? getUsageCount(glossary) : null);
}

2
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java
View File

@ -18,7 +18,6 @@ package org.openmetadata.service.jdbi3;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.schema.type.Include.ALL;
import static org.openmetadata.service.Entity.FIELD_TAGS;
import static org.openmetadata.service.Entity.GLOSSARY_TERM;
import static org.openmetadata.service.util.EntityUtil.stringMatch;
import static org.openmetadata.service.util.EntityUtil.termReferenceMatch;
@ -65,7 +64,6 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
entity.setChildren(fields.contains("children") ? getChildren(entity) : null);
entity.setRelatedTerms(fields.contains("relatedTerms") ? getRelatedTerms(entity) : null);
entity.setReviewers(fields.contains("reviewers") ? getReviewers(entity) : null);
entity.setTags(fields.contains(FIELD_TAGS) ? getTags(entity.getFullyQualifiedName()) : null);
entity.setUsageCount(fields.contains("usageCount") ? getUsageCount(entity) : null);
return entity;
}

8
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LocationRepository.java
View File

@ -14,8 +14,6 @@
package org.openmetadata.service.jdbi3;
import static org.openmetadata.service.Entity.FIELD_FOLLOWERS;
import static org.openmetadata.service.Entity.FIELD_OWNER;
import static org.openmetadata.service.Entity.FIELD_TAGS;
import static org.openmetadata.service.Entity.STORAGE_SERVICE;
import java.io.IOException;
@ -59,9 +57,7 @@ public class LocationRepository extends EntityRepository<Location> {
public Location setFields(Location location, Fields fields) throws IOException {
location.setService(getContainer(location.getId()));
location.setPath(location.getPath());
location.setOwner(fields.contains(FIELD_OWNER) ? getOwner(location) : null);
location.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(location) : null);
location.setTags(fields.contains(FIELD_TAGS) ? getTags(location.getFullyQualifiedName()) : null);
return location;
}
@ -93,7 +89,7 @@ public class LocationRepository extends EntityRepository<Location> {
List<Location> entities = new ArrayList<>();
for (String json : jsons) {
entities.add(setFields(JsonUtils.readValue(json, Location.class), fields));
entities.add(setFieldsInternal(JsonUtils.readValue(json, Location.class), fields));
}
int total =
daoCollection
@ -129,7 +125,7 @@ public class LocationRepository extends EntityRepository<Location> {
List<Location> entities = new ArrayList<>();
for (String json : jsons) {
entities.add(setFields(JsonUtils.readValue(json, Location.class), fields));
entities.add(setFieldsInternal(JsonUtils.readValue(json, Location.class), fields));
}
int total =
daoCollection

6
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java
View File

@ -15,10 +15,7 @@ package org.openmetadata.service.jdbi3;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.service.Entity.DASHBOARD;
import static org.openmetadata.service.Entity.FIELD_EXTENSION;
import static org.openmetadata.service.Entity.FIELD_FOLLOWERS;
import static org.openmetadata.service.Entity.FIELD_OWNER;
import static org.openmetadata.service.Entity.FIELD_TAGS;
import static org.openmetadata.service.Entity.MLMODEL;
import static org.openmetadata.service.Entity.MLMODEL_SERVICE;
import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch;
@ -70,14 +67,11 @@ public class MlModelRepository extends EntityRepository<MlModel> {
@Override
public MlModel setFields(MlModel mlModel, Fields fields) throws IOException {
mlModel.setOwner(fields.contains(FIELD_OWNER) ? getOwner(mlModel) : null);
mlModel.setService(getContainer(mlModel.getId()));
mlModel.setDashboard(fields.contains("dashboard") ? getDashboard(mlModel) : null);
mlModel.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(mlModel) : null);
mlModel.setTags(fields.contains(FIELD_TAGS) ? getTags(mlModel.getFullyQualifiedName()) : null);
mlModel.setUsageSummary(
fields.contains("usageSummary") ? EntityUtil.getLatestUsage(daoCollection.usageDAO(), mlModel.getId()) : null);
mlModel.setExtension(fields.contains(FIELD_EXTENSION) ? getExtension(mlModel) : null);
return mlModel;
}

6
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java
View File

@ -15,10 +15,7 @@ package org.openmetadata.service.jdbi3;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.service.Entity.FIELD_EXTENSION;
import static org.openmetadata.service.Entity.FIELD_FOLLOWERS;
import static org.openmetadata.service.Entity.FIELD_OWNER;
import static org.openmetadata.service.Entity.FIELD_TAGS;
import static org.openmetadata.service.Entity.PIPELINE_SERVICE;
import static org.openmetadata.service.util.EntityUtil.taskMatch;
@ -73,14 +70,11 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
pipeline.setPipelineUrl(pipeline.getPipelineUrl());
pipeline.setStartDate(pipeline.getStartDate());
pipeline.setConcurrency(pipeline.getConcurrency());
pipeline.setOwner(fields.contains(FIELD_OWNER) ? getOwner(pipeline) : null);
pipeline.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(pipeline) : null);
if (!fields.contains("tasks")) {
pipeline.withTasks(null);
}
pipeline.setPipelineStatus(fields.contains("pipelineStatus") ? getPipelineStatus(pipeline) : null);
pipeline.setTags(fields.contains(FIELD_TAGS) ? getTags(pipeline.getFullyQualifiedName()) : null);
pipeline.setExtension(fields.contains(FIELD_EXTENSION) ? getExtension(pipeline) : null);
return pipeline;
}

2
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PolicyRepository.java
View File

@ -169,7 +169,7 @@ public class PolicyRepository extends EntityRepository<Policy> {
List<String> jsons = daoCollection.policyDAO().listAfter(filter, Integer.MAX_VALUE, "");
List<Policy> policies = new ArrayList<>(jsons.size());
for (String json : jsons) {
Policy policy = setFields(JsonUtils.readValue(json, Policy.class), fields);
Policy policy = setFieldsInternal(JsonUtils.readValue(json, Policy.class), fields);
if (!Boolean.TRUE.equals(policy.getEnabled())) {
continue;
}

19
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java
View File

@ -120,11 +120,9 @@ public class TableRepository extends EntityRepository<Table> {
public Table setFields(Table table, Fields fields) throws IOException {
setDefaultFields(table);
table.setTableConstraints(fields.contains("tableConstraints") ? table.getTableConstraints() : null);
table.setOwner(fields.contains(FIELD_OWNER) ? getOwner(table) : null);
table.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(table) : null);
table.setUsageSummary(
fields.contains("usageSummary") ? EntityUtil.getLatestUsage(daoCollection.usageDAO(), table.getId()) : null);
table.setTags(fields.contains(FIELD_TAGS) ? getTags(table.getFullyQualifiedName()) : null);
getColumnTags(fields.contains(FIELD_TAGS), table.getColumns());
table.setJoins(fields.contains("joins") ? getJoins(table) : null);
table.setSampleData(fields.contains("sampleData") ? getSampleData(table) : null);
@ -135,7 +133,6 @@ public class TableRepository extends EntityRepository<Table> {
table.setLocation(fields.contains("location") ? getLocation(table) : null);
table.setTableQueries(fields.contains("tableQueries") ? getQueries(table) : null);
getCustomMetrics(fields.contains("customMetrics"), table);
table.setExtension(fields.contains("extension") ? getExtension(table) : null);
return table;
}
@ -215,7 +212,7 @@ public class TableRepository extends EntityRepository<Table> {
daoCollection
.entityExtensionDAO()
.insert(tableId.toString(), TABLE_SAMPLE_DATA_EXTENSION, "tableData", JsonUtils.pojoToJson(tableData));
setFields(table, Fields.EMPTY_FIELDS);
setFieldsInternal(table, Fields.EMPTY_FIELDS);
return table.withSampleData(tableData);
}
@ -251,7 +248,7 @@ public class TableRepository extends EntityRepository<Table> {
TABLE_PROFILER_CONFIG_EXTENSION,
"tableProfilerConfig",
JsonUtils.pojoToJson(tableProfilerConfig));
setFields(table, Fields.EMPTY_FIELDS);
setFieldsInternal(table, Fields.EMPTY_FIELDS);
return table.withTableProfilerConfig(tableProfilerConfig);
}
@ -261,7 +258,7 @@ public class TableRepository extends EntityRepository<Table> {
Table table = dao.findEntityById(tableId);
daoCollection.entityExtensionDAO().delete(tableId.toString(), TABLE_PROFILER_CONFIG_EXTENSION);
setFields(table, Fields.EMPTY_FIELDS);
setFieldsInternal(table, Fields.EMPTY_FIELDS);
return table;
}
@ -329,7 +326,7 @@ public class TableRepository extends EntityRepository<Table> {
JsonUtils.pojoToJson(columnProfile));
}
}
setFields(table, Fields.EMPTY_FIELDS);
setFieldsInternal(table, Fields.EMPTY_FIELDS);
return table.withProfile(createTableProfile.getTableProfile());
}
@ -362,7 +359,7 @@ public class TableRepository extends EntityRepository<Table> {
// A table has only one location.
deleteFrom(tableId, TABLE, Relationship.HAS, LOCATION);
addRelationship(tableId, locationId, TABLE, LOCATION, Relationship.HAS);
setFields(table, Fields.EMPTY_FIELDS);
setFieldsInternal(table, Fields.EMPTY_FIELDS);
return table.withLocation(location);
}
@ -388,7 +385,7 @@ public class TableRepository extends EntityRepository<Table> {
daoCollection
.entityExtensionDAO()
.insert(tableId.toString(), "table.tableQueries", "sqlQuery", JsonUtils.pojoToJson(updatedQueries));
setFields(table, Fields.EMPTY_FIELDS);
setFieldsInternal(table, Fields.EMPTY_FIELDS);
return table.withTableQueries(getQueries(table));
}
@ -420,7 +417,7 @@ public class TableRepository extends EntityRepository<Table> {
daoCollection
.entityExtensionDAO()
.insert(table.getId().toString(), extension, "customMetric", JsonUtils.pojoToJson(updatedMetrics));
setFields(table, Fields.EMPTY_FIELDS);
setFieldsInternal(table, Fields.EMPTY_FIELDS);
// return the newly created/updated custom metric only
for (Column column : table.getColumns()) {
if (column.getName().equals(columnName)) {
@ -496,7 +493,7 @@ public class TableRepository extends EntityRepository<Table> {
}
dao.update(table.getId(), JsonUtils.pojoToJson(table));
setFields(table, new Fields(List.of(FIELD_OWNER), FIELD_OWNER));
setFieldsInternal(table, new Fields(List.of(FIELD_OWNER), FIELD_OWNER));
return table;
}

2
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagCategoryRepository.java
View File

@ -69,7 +69,7 @@ public class TagCategoryRepository extends EntityRepository<TagCategory> {
List<Tag> tagList = new ArrayList<>();
for (String json : groupJsons) {
Tag tag = tagRepository.setFields(JsonUtils.readValue(json, Tag.class), fields);
Tag tag = tagRepository.setFieldsInternal(JsonUtils.readValue(json, Tag.class), fields);
tagList.add(tagRepository.populateChildrenTags(tag, fields));
}
category.withChildren(tagList.isEmpty() ? null : tagList);

2
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java
View File

@ -68,7 +68,7 @@ public class TagRepository extends EntityRepository<Tag> {
// Get tags under the given tag
List<Tag> tagList = new ArrayList<>();
for (String json : listOrEmpty(tagJsons)) {
Tag childTag = setFields(JsonUtils.readValue(json, Tag.class), fields);
Tag childTag = setFieldsInternal(JsonUtils.readValue(json, Tag.class), fields);
tagList.add(populateChildrenTags(childTag, fields));
}
return tag.withChildren(!tagList.isEmpty() ? tagList : null);

3
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java
View File

@ -151,9 +151,8 @@ public class TestCaseRepository extends EntityRepository<TestCase> {
TESTCASE_RESULT_EXTENSION,
"testCaseResult",
JsonUtils.pojoToJson(testCaseResult));
setFields(testCase, EntityUtil.Fields.EMPTY_FIELDS);
}
setFields(testCase, new EntityUtil.Fields(allowedFields, "testSuite"));
setFieldsInternal(testCase, new EntityUtil.Fields(allowedFields, "testSuite"));
ChangeDescription change =
addTestCaseChangeDescription(testCase.getVersion(), testCaseResult, storedTestCaseResult);
ChangeEvent changeEvent = getChangeEvent(withHref(uriInfo, testCase), change, entityType, testCase.getVersion());

8
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java
View File

@ -13,10 +13,7 @@
package org.openmetadata.service.jdbi3;
import static org.openmetadata.service.Entity.FIELD_EXTENSION;
import static org.openmetadata.service.Entity.FIELD_FOLLOWERS;
import static org.openmetadata.service.Entity.FIELD_OWNER;
import static org.openmetadata.service.Entity.FIELD_TAGS;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
@ -93,11 +90,8 @@ public class TopicRepository extends EntityRepository<Topic> {
@Override
public Topic setFields(Topic topic, Fields fields) throws IOException {
topic.setService(getContainer(topic.getId()));
topic.setOwner(fields.contains(FIELD_OWNER) ? getOwner(topic) : null);
topic.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(topic) : null);
topic.setTags(fields.contains(FIELD_TAGS) ? getTags(topic.getFullyQualifiedName()) : null);
topic.setSampleData(fields.contains("sampleData") ? getSampleData(topic) : null);
topic.setExtension(fields.contains(FIELD_EXTENSION) ? getExtension(topic) : null);
return topic;
}
@ -127,7 +121,7 @@ public class TopicRepository extends EntityRepository<Topic> {
daoCollection
.entityExtensionDAO()
.insert(topicId.toString(), "topic.sampleData", "topicSampleData", JsonUtils.pojoToJson(sampleData));
setFields(topic, Fields.EMPTY_FIELDS);
setFieldsInternal(topic, Fields.EMPTY_FIELDS);
return topic.withSampleData(sampleData);
}

4
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java
View File

@ -105,7 +105,7 @@ public class TypeRepository extends EntityRepository<Type> {
if (type.getCategory().equals(Category.Field)) {
throw new IllegalArgumentException("Only entity types can be extended and field types can't be extended");
}
setFields(type, putFields);
setFieldsInternal(type, putFields);
dao.findEntityById(property.getPropertyType().getId()); // Validate customProperty type exists
@ -196,7 +196,7 @@ public class TypeRepository extends EntityRepository<Type> {
daoCollection.entityExtensionDAO().deleteExtension(customPropertyFQN);
}
// Record changes to updated custome properties (only description can be updated)
// Record changes to updated custom properties (only description can be updated)
for (CustomProperty updated : updatedFields) {
// Find property that matches name and type
CustomProperty stored =

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java
View File

@ -90,7 +90,7 @@ public class DashboardResource extends EntityResource<Dashboard, DashboardReposi
}
}
static final String FIELDS = "owner,charts,followers,tags,usageSummary";
static final String FIELDS = "owner,charts,followers,tags,usageSummary,extension";
@GET
@Valid

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java
View File

@ -93,7 +93,7 @@ public class MlModelResource extends EntityResource<MlModel, MlModelRepository>
}
}
static final String FIELDS = "owner,dashboard,followers,tags,usageSummary";
static final String FIELDS = "owner,dashboard,followers,tags,usageSummary,extension";
@GET
@Valid

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java
View File

@ -106,7 +106,7 @@ public class PipelineResource extends EntityResource<Pipeline, PipelineRepositor
}
}
static final String FIELDS = "owner,tasks,pipelineStatus,followers,tags";
static final String FIELDS = "owner,tasks,pipelineStatus,followers,tags,extension";
@GET
@Valid

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java
View File

@ -93,7 +93,7 @@ public class TopicResource extends EntityResource<Topic, TopicRepository> {
}
}
static final String FIELDS = "owner,followers,tags,sampleData";
static final String FIELDS = "owner,followers,tags,sampleData,extension";
@GET
@Operation(

7
openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java
View File

@ -388,13 +388,18 @@ public final class EntityUtil {
: FullyQualifiedName.build("rules", rule.getName(), ruleField);
}
/** Return customer property field name of format "extension".propertyName */
/** Return customer property field name of format "customProperties".propertyName */
public static String getCustomField(CustomProperty property, String propertyFieldName) {
return propertyFieldName == null
? FullyQualifiedName.build("customProperties", property.getName())
: FullyQualifiedName.build("customProperties", property.getName(), propertyFieldName);
}
/** Return extension field name of format "extension".fieldName */
public static String getExtensionField(String key) {
return FullyQualifiedName.build("extension", key);
}
public static Double nextVersion(Double version) {
return Math.round((version + 0.1) * 10.0) / 10.0;
}

14
openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java
View File

@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.fasterxml.jackson.datatype.jsr353.JSR353Module;
import com.networknt.schema.JsonSchema;
@ -429,4 +430,17 @@ public final class JsonUtils {
String jsonString = EXPOSED_OBJECT_MAPPER.writeValueAsString(entity);
return EXPOSED_OBJECT_MAPPER.readValue(jsonString, clazz);
}
public static ObjectNode getObjectNode(String key, JsonNode value) {
ObjectNode objectNode = getObjectNode();
return objectNode.set(key, value);
}
public static ObjectNode getObjectNode() {
return OBJECT_MAPPER.createObjectNode();
}
public static JsonNode readTree(String extensionJson) throws JsonProcessingException {
return OBJECT_MAPPER.readTree(extensionJson);
}
}

37
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -1308,24 +1308,27 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
ObjectNode jsonNode = mapper.createObjectNode();
jsonNode.set("intA", mapper.convertValue(1, JsonNode.class));
K create = createRequest(test).withExtension(jsonNode);
createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
T entity = createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
// PUT and update the entity with extension field intA to a new value
// TODO to do change description for stored customProperties
jsonNode.set("intA", mapper.convertValue(2, JsonNode.class));
JsonNode intAValue = mapper.convertValue(2, JsonNode.class);
jsonNode.set("intA", intAValue);
create = createRequest(test).withExtension(jsonNode);
T entity = updateEntity(create, Status.OK, ADMIN_AUTH_HEADERS);
assertEquals(JsonUtils.valueToTree(create.getExtension()), JsonUtils.valueToTree(entity.getExtension()));
change = getChangeDescription(entity.getVersion());
fieldUpdated(change, EntityUtil.getExtensionField("intA"), mapper.convertValue(1, JsonNode.class), intAValue);
entity = updateAndCheckEntity(create, Status.OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
// PATCH and update the entity with extension field stringB
// TODO to do change description for stored customProperties
json = JsonUtils.pojoToJson(entity);
jsonNode.set("stringB", mapper.convertValue("stringB", JsonNode.class));
JsonNode stringBValue = mapper.convertValue("stringB", JsonNode.class);
jsonNode.set("stringB", stringBValue);
entity.setExtension(jsonNode);
entity = patchEntity(entity.getId(), json, entity, ADMIN_AUTH_HEADERS);
change = getChangeDescription(entity.getVersion());
fieldAdded(change, "extension", List.of(JsonUtils.getObjectNode("stringB", stringBValue)));
entity = patchEntityAndCheck(entity, json, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
assertEquals(JsonUtils.valueToTree(jsonNode), JsonUtils.valueToTree(entity.getExtension()));
// PUT and remove field intA from the entity extension - for BOT this should be ignored
// PUT and remove field intA from the entity extension - *** for BOT this should be ignored ***
JsonNode oldNode = JsonUtils.valueToTree(entity.getExtension());
jsonNode.remove("intA");
create = createRequest(test).withExtension(jsonNode);
@ -1333,17 +1336,19 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
assertNotEquals(JsonUtils.valueToTree(create.getExtension()), JsonUtils.valueToTree(entity.getExtension()));
assertEquals(oldNode, JsonUtils.valueToTree(entity.getExtension())); // Extension remains as is
// PUT and remove field intA from the the entity extension
// TODO to do change description for stored customProperties
entity = updateEntity(create, Status.OK, ADMIN_AUTH_HEADERS);
// PUT and remove field intA from the entity extension (for non-bot this should succeed)
change = getChangeDescription(entity.getVersion());
fieldDeleted(change, "extension", List.of(JsonUtils.getObjectNode("intA", intAValue)));
entity = updateAndCheckEntity(create, Status.OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
assertEquals(JsonUtils.valueToTree(create.getExtension()), JsonUtils.valueToTree(entity.getExtension()));
// PATCH and remove field stringB from the the entity extension
// TODO to do change description for stored customProperties
// PATCH and remove field stringB from the entity extension
json = JsonUtils.pojoToJson(entity);
jsonNode.remove("stringB");
entity.setExtension(jsonNode);
entity = patchEntity(entity.getId(), json, entity, ADMIN_AUTH_HEADERS);
change = getChangeDescription(entity.getVersion());
fieldDeleted(change, "extension", List.of(JsonUtils.getObjectNode("stringB", stringBValue)));
entity = patchEntityAndCheck(entity, json, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
assertEquals(JsonUtils.valueToTree(jsonNode), JsonUtils.valueToTree(entity.getExtension()));
// Now set the entity custom property to an invalid value
@ -1982,6 +1987,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
List<TagLabel> expectedTags = (List<TagLabel>) expected;
List<TagLabel> actualTags = JsonUtils.readObjects(actual.toString(), TagLabel.class);
assertTrue(actualTags.containsAll(expectedTags));
} else if (fieldName.startsWith("extension")) { // Custom properties related extension field changes
assertEquals(expected.toString(), actual.toString());
} else {
// All the other fields
assertEquals(expected, actual, "Field name " + fieldName);