diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ChartRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ChartRepository.java index c0db23f68ae..aed497582b0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ChartRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ChartRepository.java @@ -14,8 +14,6 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; -import static org.openmetadata.service.Entity.FIELD_OWNER; -import static org.openmetadata.service.Entity.FIELD_TAGS; import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; @@ -91,9 +89,7 @@ public class ChartRepository extends EntityRepository { @Override public Chart setFields(Chart chart, Fields fields) throws IOException { chart.setService(getContainer(chart.getId())); - chart.setOwner(fields.contains(FIELD_OWNER) ? getOwner(chart) : null); chart.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(chart) : null); - chart.setTags(fields.contains(FIELD_TAGS) ? getTags(chart.getFullyQualifiedName()) : null); return chart; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardRepository.java index e32c953e2a1..1ccdcf6c465 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardRepository.java @@ -14,10 +14,7 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; -import static org.openmetadata.service.Entity.FIELD_EXTENSION; import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; -import static org.openmetadata.service.Entity.FIELD_OWNER; -import static org.openmetadata.service.Entity.FIELD_TAGS; import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; @@ -61,15 +58,12 @@ public class DashboardRepository extends EntityRepository { @Override public Dashboard setFields(Dashboard dashboard, Fields fields) throws IOException { dashboard.setService(getContainer(dashboard.getId())); - dashboard.setOwner(fields.contains(FIELD_OWNER) ? getOwner(dashboard) : null); dashboard.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(dashboard) : null); dashboard.setCharts(fields.contains("charts") ? getCharts(dashboard) : null); - dashboard.setTags(fields.contains(FIELD_TAGS) ? getTags(dashboard.getFullyQualifiedName()) : null); dashboard.setUsageSummary( fields.contains("usageSummary") ? EntityUtil.getLatestUsage(daoCollection.usageDAO(), dashboard.getId()) : null); - dashboard.setExtension(fields.contains(FIELD_EXTENSION) ? getExtension(dashboard) : null); return dashboard; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index c5642b07dc1..bc88d8b47bc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -29,6 +29,7 @@ import static org.openmetadata.service.util.EntityUtil.compareTagLabel; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; import static org.openmetadata.service.util.EntityUtil.fieldAdded; import static org.openmetadata.service.util.EntityUtil.fieldDeleted; +import static org.openmetadata.service.util.EntityUtil.getExtensionField; import static org.openmetadata.service.util.EntityUtil.nextMajorVersion; import static org.openmetadata.service.util.EntityUtil.nextVersion; import static org.openmetadata.service.util.EntityUtil.objectMatch; @@ -36,7 +37,6 @@ import static org.openmetadata.service.util.EntityUtil.tagLabelMatch; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.networknt.schema.JsonSchema; import com.networknt.schema.ValidationMessage; @@ -283,7 +283,7 @@ public abstract class EntityRepository { @Transaction public final T get(UriInfo uriInfo, UUID id, Fields fields, Include include) throws IOException { - return withHref(uriInfo, setFields(dao.findEntityById(id, include), fields)); + return withHref(uriInfo, setFieldsInternal(dao.findEntityById(id, include), fields)); } @Transaction @@ -293,7 +293,7 @@ public abstract class EntityRepository { @Transaction public final T getByName(UriInfo uriInfo, String fqn, Fields fields, Include include) throws IOException { - return withHref(uriInfo, setFields(dao.findEntityByName(fqn, include), fields)); + return withHref(uriInfo, setFieldsInternal(dao.findEntityByName(fqn, include), fields)); } @Transaction @@ -306,7 +306,7 @@ public abstract class EntityRepository { List jsons = dao.listAfter(filter, limitParam + 1, after == null ? "" : RestUtil.decodeCursor(after)); for (String json : jsons) { - T entity = withHref(uriInfo, setFields(JsonUtils.readValue(json, entityClass), fields)); + T entity = withHref(uriInfo, setFieldsInternal(JsonUtils.readValue(json, entityClass), fields)); entities.add(entity); } @@ -332,7 +332,7 @@ public abstract class EntityRepository { List entities = new ArrayList<>(); for (String json : jsons) { - T entity = withHref(uriInfo, setFields(JsonUtils.readValue(json, entityClass), fields)); + T entity = withHref(uriInfo, setFieldsInternal(JsonUtils.readValue(json, entityClass), fields)); entities.add(entity); } int total = dao.listCount(filter); @@ -358,7 +358,7 @@ public abstract class EntityRepository { return JsonUtils.readValue(json, entityClass); } // If requested the latest version, return it from current version of the entity - T entity = setFields(dao.findEntityById(id, ALL), putFields); + T entity = setFieldsInternal(dao.findEntityById(id, ALL), putFields); if (entity.getVersion().equals(requestedVersion)) { return entity; } @@ -368,7 +368,7 @@ public abstract class EntityRepository { @Transaction public EntityHistory listVersions(UUID id) throws IOException { - T latest = setFields(dao.findEntityById(id, ALL), putFields); + T latest = setFieldsInternal(dao.findEntityById(id, ALL), putFields); String extensionPrefix = EntityUtil.getVersionExtensionPrefix(entityType); List records = daoCollection.entityExtensionDAO().getExtensions(id.toString(), extensionPrefix); List oldVersions = new ArrayList<>(); @@ -389,15 +389,26 @@ public abstract class EntityRepository { @Transaction public final T createInternal(T entity) throws IOException { + prepareInternal(entity); + return createNewEntity(entity); + } + + private void prepareInternal(T entity) throws IOException { prepare(entity); validateExtension(entity); - return createNewEntity(entity); + } + + T setFieldsInternal(T entity, Fields fields) throws IOException { + entity.setOwner(fields.contains(FIELD_OWNER) ? getOwner(entity) : null); + entity.setTags(fields.contains(FIELD_TAGS) ? getTags(entity.getFullyQualifiedName()) : null); + entity.setExtension(fields.contains("extension") ? getExtension(entity) : null); + setFields(entity, fields); + return entity; } @Transaction public final PutResponse createOrUpdate(UriInfo uriInfo, T original, T updated) throws IOException { - prepare(updated); - validateExtension(updated); + prepareInternal(updated); // Check if there is any original, deleted or not original = JsonUtils.readValue(dao.findJsonByFqn(original.getFullyQualifiedName(), ALL), entityClass); if (original == null) { @@ -418,7 +429,6 @@ public abstract class EntityRepository { @Transaction public final PutResponse createOrUpdateInternal(UriInfo uriInfo, T updated) throws IOException { - validateExtension(updated); // Check if there is any original, deleted or not T original = JsonUtils.readValue(dao.findJsonByFqn(updated.getFullyQualifiedName(), ALL), entityClass); if (original == null) { @@ -440,7 +450,7 @@ public abstract class EntityRepository { @Transaction public PutResponse update(UriInfo uriInfo, T original, T updated) throws IOException { // Get all the fields in the original entity that can be updated during PUT operation - setFields(original, putFields); + setFieldsInternal(original, putFields); // If the entity state is soft-deleted, recursively undelete the entity and it's children if (Boolean.TRUE.equals(original.getDeleted())) { @@ -457,16 +467,15 @@ public abstract class EntityRepository { @Transaction public final PatchResponse patch(UriInfo uriInfo, UUID id, String user, JsonPatch patch) throws IOException { // Get all the fields in the original entity that can be updated during PATCH operation - T original = setFields(dao.findEntityById(id), patchFields); + T original = setFieldsInternal(dao.findEntityById(id), patchFields); // Apply JSON patch to the original entity to get the updated entity T updated = JsonUtils.applyPatch(original, patch, entityClass); updated.setUpdatedBy(user); updated.setUpdatedAt(System.currentTimeMillis()); - prepare(updated); + prepareInternal(updated); populateOwner(updated.getOwner()); - validateExtension(updated); restorePatchAttributes(original, updated); // Update the attributes and relationships of an entity @@ -537,13 +546,13 @@ public abstract class EntityRepository { throws IOException { T original = JsonUtils.readValue(json, entityClass); preDelete(original); - setFields(original, putFields); + setFieldsInternal(original, putFields); deleteChildren(id, recursive, hardDelete, updatedBy); String changeType; T updated = JsonUtils.readValue(json, entityClass); - setFields(updated, putFields); // we need service, database, databaseSchema to delete properly from ES. + setFieldsInternal(updated, putFields); // we need service, database, databaseSchema to delete properly from ES. if (supportsSoftDelete && !hardDelete) { updated.setUpdatedBy(updatedBy); updated.setUpdatedAt(System.currentTimeMillis()); @@ -739,11 +748,10 @@ public abstract class EntityRepository { if (records.isEmpty()) { return null; } - ObjectMapper mapper = new ObjectMapper(); - ObjectNode objectNode = mapper.createObjectNode(); + ObjectNode objectNode = JsonUtils.getObjectNode(); for (ExtensionRecord record : records) { String fieldName = TypeRegistry.getPropertyName(record.getExtensionName()); - objectNode.set(fieldName, mapper.readTree(record.getExtensionJson())); + objectNode.set(fieldName, JsonUtils.readTree(record.getExtensionJson())); } return objectNode; } @@ -1239,12 +1247,47 @@ public abstract class EntityRepository { } private void updateExtension() throws JsonProcessingException { + if (original.getExtension() == updated.getExtension()) { + return; + } + if (updatedByBot()) { // Revert changes to extension field, if being updated by a bot updated.setExtension(original.getExtension()); return; } + List added = new ArrayList<>(); + List deleted = new ArrayList<>(); + JsonNode origFields = JsonUtils.valueToTree(original.getExtension()); + JsonNode updatedFields = JsonUtils.valueToTree(updated.getExtension()); + + // Check for updated and deleted fields + for (Iterator> it = origFields.fields(); it.hasNext(); ) { + Entry orig = it.next(); + JsonNode updated = updatedFields.get(orig.getKey()); + if (updated == null) { + deleted.add(JsonUtils.getObjectNode(orig.getKey(), orig.getValue())); + } else { + // TODO converting to a string is a hack for now because JsonNode equals issues + recordChange(getExtensionField(orig.getKey()), orig.getValue().toString(), updated.toString()); + } + } + + // Check for added fields + for (Iterator> it = updatedFields.fields(); it.hasNext(); ) { + Entry updated = it.next(); + JsonNode orig = origFields.get(updated.getKey()); + if (orig == null) { + added.add(JsonUtils.getObjectNode(updated.getKey(), updated.getValue())); + } + } + if (!added.isEmpty()) { + fieldAdded(changeDescription, "extension", JsonUtils.pojoToJson(added)); + } + if (!deleted.isEmpty()) { + fieldDeleted(changeDescription, "extension", JsonUtils.pojoToJson(deleted)); + } removeExtension(original); storeExtension(updated); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java index 39b521865d0..7f68e51a3bc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java @@ -17,8 +17,6 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; -import static org.openmetadata.service.Entity.FIELD_OWNER; -import static org.openmetadata.service.Entity.FIELD_TAGS; import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; @@ -51,8 +49,6 @@ public class GlossaryRepository extends EntityRepository { @Override public Glossary setFields(Glossary glossary, Fields fields) throws IOException { - glossary.setOwner(fields.contains(FIELD_OWNER) ? getOwner(glossary) : null); - glossary.setTags(fields.contains(FIELD_TAGS) ? getTags(glossary.getName()) : null); glossary.setReviewers(fields.contains("reviewers") ? getReviewers(glossary) : null); return glossary.withUsageCount(fields.contains("usageCount") ? getUsageCount(glossary) : null); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index b72f18367c4..df486497d10 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -18,7 +18,6 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.schema.type.Include.ALL; -import static org.openmetadata.service.Entity.FIELD_TAGS; import static org.openmetadata.service.Entity.GLOSSARY_TERM; import static org.openmetadata.service.util.EntityUtil.stringMatch; import static org.openmetadata.service.util.EntityUtil.termReferenceMatch; @@ -65,7 +64,6 @@ public class GlossaryTermRepository extends EntityRepository { entity.setChildren(fields.contains("children") ? getChildren(entity) : null); entity.setRelatedTerms(fields.contains("relatedTerms") ? getRelatedTerms(entity) : null); entity.setReviewers(fields.contains("reviewers") ? getReviewers(entity) : null); - entity.setTags(fields.contains(FIELD_TAGS) ? getTags(entity.getFullyQualifiedName()) : null); entity.setUsageCount(fields.contains("usageCount") ? getUsageCount(entity) : null); return entity; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LocationRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LocationRepository.java index 84fb1286af3..813093e714d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LocationRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LocationRepository.java @@ -14,8 +14,6 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; -import static org.openmetadata.service.Entity.FIELD_OWNER; -import static org.openmetadata.service.Entity.FIELD_TAGS; import static org.openmetadata.service.Entity.STORAGE_SERVICE; import java.io.IOException; @@ -59,9 +57,7 @@ public class LocationRepository extends EntityRepository { public Location setFields(Location location, Fields fields) throws IOException { location.setService(getContainer(location.getId())); location.setPath(location.getPath()); - location.setOwner(fields.contains(FIELD_OWNER) ? getOwner(location) : null); location.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(location) : null); - location.setTags(fields.contains(FIELD_TAGS) ? getTags(location.getFullyQualifiedName()) : null); return location; } @@ -93,7 +89,7 @@ public class LocationRepository extends EntityRepository { List entities = new ArrayList<>(); for (String json : jsons) { - entities.add(setFields(JsonUtils.readValue(json, Location.class), fields)); + entities.add(setFieldsInternal(JsonUtils.readValue(json, Location.class), fields)); } int total = daoCollection @@ -129,7 +125,7 @@ public class LocationRepository extends EntityRepository { List entities = new ArrayList<>(); for (String json : jsons) { - entities.add(setFields(JsonUtils.readValue(json, Location.class), fields)); + entities.add(setFieldsInternal(JsonUtils.readValue(json, Location.class), fields)); } int total = daoCollection diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java index 3939b3f7fc5..c6cb450bc03 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java @@ -15,10 +15,7 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.Entity.DASHBOARD; -import static org.openmetadata.service.Entity.FIELD_EXTENSION; import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; -import static org.openmetadata.service.Entity.FIELD_OWNER; -import static org.openmetadata.service.Entity.FIELD_TAGS; import static org.openmetadata.service.Entity.MLMODEL; import static org.openmetadata.service.Entity.MLMODEL_SERVICE; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; @@ -70,14 +67,11 @@ public class MlModelRepository extends EntityRepository { @Override public MlModel setFields(MlModel mlModel, Fields fields) throws IOException { - mlModel.setOwner(fields.contains(FIELD_OWNER) ? getOwner(mlModel) : null); mlModel.setService(getContainer(mlModel.getId())); mlModel.setDashboard(fields.contains("dashboard") ? getDashboard(mlModel) : null); mlModel.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(mlModel) : null); - mlModel.setTags(fields.contains(FIELD_TAGS) ? getTags(mlModel.getFullyQualifiedName()) : null); mlModel.setUsageSummary( fields.contains("usageSummary") ? EntityUtil.getLatestUsage(daoCollection.usageDAO(), mlModel.getId()) : null); - mlModel.setExtension(fields.contains(FIELD_EXTENSION) ? getExtension(mlModel) : null); return mlModel; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java index 8b40ab66657..b782c1bd878 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java @@ -15,10 +15,7 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import static org.openmetadata.service.Entity.FIELD_EXTENSION; import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; -import static org.openmetadata.service.Entity.FIELD_OWNER; -import static org.openmetadata.service.Entity.FIELD_TAGS; import static org.openmetadata.service.Entity.PIPELINE_SERVICE; import static org.openmetadata.service.util.EntityUtil.taskMatch; @@ -73,14 +70,11 @@ public class PipelineRepository extends EntityRepository { pipeline.setPipelineUrl(pipeline.getPipelineUrl()); pipeline.setStartDate(pipeline.getStartDate()); pipeline.setConcurrency(pipeline.getConcurrency()); - pipeline.setOwner(fields.contains(FIELD_OWNER) ? getOwner(pipeline) : null); pipeline.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(pipeline) : null); if (!fields.contains("tasks")) { pipeline.withTasks(null); } pipeline.setPipelineStatus(fields.contains("pipelineStatus") ? getPipelineStatus(pipeline) : null); - pipeline.setTags(fields.contains(FIELD_TAGS) ? getTags(pipeline.getFullyQualifiedName()) : null); - pipeline.setExtension(fields.contains(FIELD_EXTENSION) ? getExtension(pipeline) : null); return pipeline; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PolicyRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PolicyRepository.java index e56e368e897..97d8b0d096b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PolicyRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PolicyRepository.java @@ -169,7 +169,7 @@ public class PolicyRepository extends EntityRepository { List jsons = daoCollection.policyDAO().listAfter(filter, Integer.MAX_VALUE, ""); List policies = new ArrayList<>(jsons.size()); for (String json : jsons) { - Policy policy = setFields(JsonUtils.readValue(json, Policy.class), fields); + Policy policy = setFieldsInternal(JsonUtils.readValue(json, Policy.class), fields); if (!Boolean.TRUE.equals(policy.getEnabled())) { continue; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 91acf624677..b7a63539e56 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -120,11 +120,9 @@ public class TableRepository extends EntityRepository { public Table setFields(Table table, Fields fields) throws IOException { setDefaultFields(table); table.setTableConstraints(fields.contains("tableConstraints") ? table.getTableConstraints() : null); - table.setOwner(fields.contains(FIELD_OWNER) ? getOwner(table) : null); table.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(table) : null); table.setUsageSummary( fields.contains("usageSummary") ? EntityUtil.getLatestUsage(daoCollection.usageDAO(), table.getId()) : null); - table.setTags(fields.contains(FIELD_TAGS) ? getTags(table.getFullyQualifiedName()) : null); getColumnTags(fields.contains(FIELD_TAGS), table.getColumns()); table.setJoins(fields.contains("joins") ? getJoins(table) : null); table.setSampleData(fields.contains("sampleData") ? getSampleData(table) : null); @@ -135,7 +133,6 @@ public class TableRepository extends EntityRepository
{ table.setLocation(fields.contains("location") ? getLocation(table) : null); table.setTableQueries(fields.contains("tableQueries") ? getQueries(table) : null); getCustomMetrics(fields.contains("customMetrics"), table); - table.setExtension(fields.contains("extension") ? getExtension(table) : null); return table; } @@ -215,7 +212,7 @@ public class TableRepository extends EntityRepository
{ daoCollection .entityExtensionDAO() .insert(tableId.toString(), TABLE_SAMPLE_DATA_EXTENSION, "tableData", JsonUtils.pojoToJson(tableData)); - setFields(table, Fields.EMPTY_FIELDS); + setFieldsInternal(table, Fields.EMPTY_FIELDS); return table.withSampleData(tableData); } @@ -251,7 +248,7 @@ public class TableRepository extends EntityRepository
{ TABLE_PROFILER_CONFIG_EXTENSION, "tableProfilerConfig", JsonUtils.pojoToJson(tableProfilerConfig)); - setFields(table, Fields.EMPTY_FIELDS); + setFieldsInternal(table, Fields.EMPTY_FIELDS); return table.withTableProfilerConfig(tableProfilerConfig); } @@ -261,7 +258,7 @@ public class TableRepository extends EntityRepository
{ Table table = dao.findEntityById(tableId); daoCollection.entityExtensionDAO().delete(tableId.toString(), TABLE_PROFILER_CONFIG_EXTENSION); - setFields(table, Fields.EMPTY_FIELDS); + setFieldsInternal(table, Fields.EMPTY_FIELDS); return table; } @@ -329,7 +326,7 @@ public class TableRepository extends EntityRepository
{ JsonUtils.pojoToJson(columnProfile)); } } - setFields(table, Fields.EMPTY_FIELDS); + setFieldsInternal(table, Fields.EMPTY_FIELDS); return table.withProfile(createTableProfile.getTableProfile()); } @@ -362,7 +359,7 @@ public class TableRepository extends EntityRepository
{ // A table has only one location. deleteFrom(tableId, TABLE, Relationship.HAS, LOCATION); addRelationship(tableId, locationId, TABLE, LOCATION, Relationship.HAS); - setFields(table, Fields.EMPTY_FIELDS); + setFieldsInternal(table, Fields.EMPTY_FIELDS); return table.withLocation(location); } @@ -388,7 +385,7 @@ public class TableRepository extends EntityRepository
{ daoCollection .entityExtensionDAO() .insert(tableId.toString(), "table.tableQueries", "sqlQuery", JsonUtils.pojoToJson(updatedQueries)); - setFields(table, Fields.EMPTY_FIELDS); + setFieldsInternal(table, Fields.EMPTY_FIELDS); return table.withTableQueries(getQueries(table)); } @@ -420,7 +417,7 @@ public class TableRepository extends EntityRepository
{ daoCollection .entityExtensionDAO() .insert(table.getId().toString(), extension, "customMetric", JsonUtils.pojoToJson(updatedMetrics)); - setFields(table, Fields.EMPTY_FIELDS); + setFieldsInternal(table, Fields.EMPTY_FIELDS); // return the newly created/updated custom metric only for (Column column : table.getColumns()) { if (column.getName().equals(columnName)) { @@ -496,7 +493,7 @@ public class TableRepository extends EntityRepository
{ } dao.update(table.getId(), JsonUtils.pojoToJson(table)); - setFields(table, new Fields(List.of(FIELD_OWNER), FIELD_OWNER)); + setFieldsInternal(table, new Fields(List.of(FIELD_OWNER), FIELD_OWNER)); return table; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagCategoryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagCategoryRepository.java index 1145bfc90b1..14f6fa4ef2d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagCategoryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagCategoryRepository.java @@ -69,7 +69,7 @@ public class TagCategoryRepository extends EntityRepository { List tagList = new ArrayList<>(); for (String json : groupJsons) { - Tag tag = tagRepository.setFields(JsonUtils.readValue(json, Tag.class), fields); + Tag tag = tagRepository.setFieldsInternal(JsonUtils.readValue(json, Tag.class), fields); tagList.add(tagRepository.populateChildrenTags(tag, fields)); } category.withChildren(tagList.isEmpty() ? null : tagList); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java index 0e6b23a6d32..9fa15f25575 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java @@ -68,7 +68,7 @@ public class TagRepository extends EntityRepository { // Get tags under the given tag List tagList = new ArrayList<>(); for (String json : listOrEmpty(tagJsons)) { - Tag childTag = setFields(JsonUtils.readValue(json, Tag.class), fields); + Tag childTag = setFieldsInternal(JsonUtils.readValue(json, Tag.class), fields); tagList.add(populateChildrenTags(childTag, fields)); } return tag.withChildren(!tagList.isEmpty() ? tagList : null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java index dabd38036c8..0d3c6ebf5c6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java @@ -151,9 +151,8 @@ public class TestCaseRepository extends EntityRepository { TESTCASE_RESULT_EXTENSION, "testCaseResult", JsonUtils.pojoToJson(testCaseResult)); - setFields(testCase, EntityUtil.Fields.EMPTY_FIELDS); } - setFields(testCase, new EntityUtil.Fields(allowedFields, "testSuite")); + setFieldsInternal(testCase, new EntityUtil.Fields(allowedFields, "testSuite")); ChangeDescription change = addTestCaseChangeDescription(testCase.getVersion(), testCaseResult, storedTestCaseResult); ChangeEvent changeEvent = getChangeEvent(withHref(uriInfo, testCase), change, entityType, testCase.getVersion()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java index f022ce90332..ff57a91cd19 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java @@ -13,10 +13,7 @@ package org.openmetadata.service.jdbi3; -import static org.openmetadata.service.Entity.FIELD_EXTENSION; import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; -import static org.openmetadata.service.Entity.FIELD_OWNER; -import static org.openmetadata.service.Entity.FIELD_TAGS; import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; @@ -93,11 +90,8 @@ public class TopicRepository extends EntityRepository { @Override public Topic setFields(Topic topic, Fields fields) throws IOException { topic.setService(getContainer(topic.getId())); - topic.setOwner(fields.contains(FIELD_OWNER) ? getOwner(topic) : null); topic.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(topic) : null); - topic.setTags(fields.contains(FIELD_TAGS) ? getTags(topic.getFullyQualifiedName()) : null); topic.setSampleData(fields.contains("sampleData") ? getSampleData(topic) : null); - topic.setExtension(fields.contains(FIELD_EXTENSION) ? getExtension(topic) : null); return topic; } @@ -127,7 +121,7 @@ public class TopicRepository extends EntityRepository { daoCollection .entityExtensionDAO() .insert(topicId.toString(), "topic.sampleData", "topicSampleData", JsonUtils.pojoToJson(sampleData)); - setFields(topic, Fields.EMPTY_FIELDS); + setFieldsInternal(topic, Fields.EMPTY_FIELDS); return topic.withSampleData(sampleData); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java index 6c26cfd1c6f..f3e6fc27612 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java @@ -105,7 +105,7 @@ public class TypeRepository extends EntityRepository { if (type.getCategory().equals(Category.Field)) { throw new IllegalArgumentException("Only entity types can be extended and field types can't be extended"); } - setFields(type, putFields); + setFieldsInternal(type, putFields); dao.findEntityById(property.getPropertyType().getId()); // Validate customProperty type exists @@ -196,7 +196,7 @@ public class TypeRepository extends EntityRepository { daoCollection.entityExtensionDAO().deleteExtension(customPropertyFQN); } - // Record changes to updated custome properties (only description can be updated) + // Record changes to updated custom properties (only description can be updated) for (CustomProperty updated : updatedFields) { // Find property that matches name and type CustomProperty stored = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java index 0ea93488028..365231254f9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java @@ -90,7 +90,7 @@ public class DashboardResource extends EntityResource } } - static final String FIELDS = "owner,dashboard,followers,tags,usageSummary"; + static final String FIELDS = "owner,dashboard,followers,tags,usageSummary,extension"; @GET @Valid diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java index 3edb4798dfe..8ee8b96c5f1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java @@ -106,7 +106,7 @@ public class PipelineResource extends EntityResource { } } - static final String FIELDS = "owner,followers,tags,sampleData"; + static final String FIELDS = "owner,followers,tags,sampleData,extension"; @GET @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java index afd3f36f262..b2cc6ccfb08 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java @@ -388,13 +388,18 @@ public final class EntityUtil { : FullyQualifiedName.build("rules", rule.getName(), ruleField); } - /** Return customer property field name of format "extension".propertyName */ + /** Return customer property field name of format "customProperties".propertyName */ public static String getCustomField(CustomProperty property, String propertyFieldName) { return propertyFieldName == null ? FullyQualifiedName.build("customProperties", property.getName()) : FullyQualifiedName.build("customProperties", property.getName(), propertyFieldName); } + /** Return extension field name of format "extension".fieldName */ + public static String getExtensionField(String key) { + return FullyQualifiedName.build("extension", key); + } + public static Double nextVersion(Double version) { return Math.round((version + 0.1) * 10.0) / 10.0; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java index 732d8e58ec8..6f28652a74c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.type.TypeFactory; import com.fasterxml.jackson.datatype.jsr353.JSR353Module; import com.networknt.schema.JsonSchema; @@ -429,4 +430,17 @@ public final class JsonUtils { String jsonString = EXPOSED_OBJECT_MAPPER.writeValueAsString(entity); return EXPOSED_OBJECT_MAPPER.readValue(jsonString, clazz); } + + public static ObjectNode getObjectNode(String key, JsonNode value) { + ObjectNode objectNode = getObjectNode(); + return objectNode.set(key, value); + } + + public static ObjectNode getObjectNode() { + return OBJECT_MAPPER.createObjectNode(); + } + + public static JsonNode readTree(String extensionJson) throws JsonProcessingException { + return OBJECT_MAPPER.readTree(extensionJson); + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index d71654b84d5..cda2baa7748 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -1308,24 +1308,27 @@ public abstract class EntityResourceTest expectedTags = (List) expected; List actualTags = JsonUtils.readObjects(actual.toString(), TagLabel.class); assertTrue(actualTags.containsAll(expectedTags)); + } else if (fieldName.startsWith("extension")) { // Custom properties related extension field changes + assertEquals(expected.toString(), actual.toString()); } else { // All the other fields assertEquals(expected, actual, "Field name " + fieldName);