diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql index 60db0ce87e9..2c6bf5877eb 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql @@ -60,3 +60,7 @@ SET json = JSON_REPLACE( ) where de2.serviceType = 'Mssql' and JSON_EXTRACT(json, '$.connection.config.database') is NULL; + +-- column deleted not needed for entities that don't support soft delete +ALTER TABLE query_entity DROP COLUMN deleted; +ALTER TABLE event_subscription_entity DROP COLUMN deleted; diff --git a/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql index dacb509d172..7077980cec9 100644 --- a/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql @@ -37,7 +37,7 @@ where json#>>'{connection,config,scheme}' in ('impala', 'impala4'); -- remove the dataModel references from Data Models UPDATE dashboard_data_model_entity SET json = json #- '{dataModels}'; --- migrate ingestAllDatabases in mssql +-- migrate ingestAllDatabases in mssql UPDATE dbservice_entity de2 SET json = JSONB_SET( json || JSONB_SET(json,'{connection,config}', json#>'{connection,config}'|| @@ -56,3 +56,7 @@ SET json = JSONB_SET( ) WHERE de2.serviceType = 'Mssql' AND json->>'{connection,config,database}' IS NULL; + +-- column deleted not needed for entities that don't support soft delete +ALTER TABLE query_entity DROP COLUMN deleted; +ALTER TABLE event_subscription_entity DROP COLUMN deleted; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 713675ed116..e63cebf221e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -1424,6 +1424,11 @@ public interface CollectionDAO { @SqlQuery("SELECT json FROM ") List listAllEventsSubscriptions(@Define("table") String table); + + @Override + default boolean supportsSoftDelete() { + return false; + } } interface ChartDAO extends EntityDAO { @@ -1800,6 +1805,11 @@ public interface CollectionDAO { return "nameHash"; } + @Override + default boolean supportsSoftDelete() { + return false; + } + @Override default int listCount(ListFilter filter) { String entityId = filter.getQueryParam("entityId"); @@ -3267,15 +3277,6 @@ public interface CollectionDAO { + "ORDER BY timestamp DESC LIMIT 1") String getLatestExtensionByFQN(@Bind("entityFQNHash") String entityFQNHash, @Bind("jsonSchema") String jsonSchema); - @SqlQuery( - "SELECT json FROM entity_extension_time_series where entityFQNHash = :entityFQNHash and jsonSchema = :jsonSchema " - + " AND timestamp >= :startTs and timestamp <= :endTs ORDER BY timestamp DESC") - List listBetweenTimestampsByFQN( - @Bind("entityFQNHash") String entityFQNHash, - @Bind("jsonSchema") String jsonSchema, - @Bind("startTs") Long startTs, - @Bind("endTs") long endTs); - @SqlQuery( "SELECT json FROM entity_extension_time_series where entityFQNHash = :entityFQNHash and extension = :extension " + " AND timestamp >= :startTs and timestamp <= :endTs ORDER BY timestamp DESC") diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java index 3a506ad8494..de6560a83a8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java @@ -15,7 +15,6 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.service.Entity.DOMAIN; -import static org.openmetadata.service.Entity.GLOSSARY_TERM; import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; @@ -59,7 +58,7 @@ public class DomainRepository extends EntityRepository { private List getChildren(Domain entity) throws IOException { List ids = findTo(entity.getId(), DOMAIN, Relationship.CONTAINS, DOMAIN); - return EntityUtil.populateEntityReferences(ids, GLOSSARY_TERM); + return EntityUtil.populateEntityReferences(ids, DOMAIN); } private List getExperts(Domain entity) throws IOException { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 01a9173609a..b9c665e2141 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -1712,21 +1712,21 @@ public abstract class EntityRepository { // Check for updated and deleted fields for (Iterator> it = origFields.fields(); it.hasNext(); ) { Entry orig = it.next(); - JsonNode updated = updatedFields.get(orig.getKey()); - if (updated == null) { + JsonNode updatedField = updatedFields.get(orig.getKey()); + if (updatedField == null) { deleted.add(JsonUtils.getObjectNode(orig.getKey(), orig.getValue())); } else { // TODO converting to a string is a hack for now because JsonNode equals issues - recordChange(getExtensionField(orig.getKey()), orig.getValue().toString(), updated.toString()); + recordChange(getExtensionField(orig.getKey()), orig.getValue().toString(), updatedField.toString()); } } // Check for added fields for (Iterator> it = updatedFields.fields(); it.hasNext(); ) { - Entry updated = it.next(); - JsonNode orig = origFields.get(updated.getKey()); + Entry updatedField = it.next(); + JsonNode orig = origFields.get(updatedField.getKey()); if (orig == null) { - added.add(JsonUtils.getObjectNode(updated.getKey(), updated.getValue())); + added.add(JsonUtils.getObjectNode(updatedField.getKey(), updatedField.getValue())); } } if (!added.isEmpty()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java index 2e71f58d80e..d842bdd8e50 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java @@ -58,7 +58,6 @@ import org.openmetadata.schema.entity.events.EventSubscription; import org.openmetadata.schema.entity.events.SubscriptionStatus; import org.openmetadata.schema.type.EntityHistory; import org.openmetadata.schema.type.Function; -import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.SubscriptionResourceDescriptor; import org.openmetadata.service.Entity; @@ -192,15 +191,9 @@ public class EventSubscriptionResource extends EntityResource { String before, @Parameter(description = "Returns list of queries after this cursor", schema = @Schema(type = "string")) @QueryParam("after") - String after, - @Parameter( - description = "Include all, deleted, or non-deleted entities.", - schema = @Schema(implementation = Include.class)) - @QueryParam("include") - @DefaultValue("non-deleted") - Include include) + String after) throws IOException { - ListFilter filter = new ListFilter(include); + ListFilter filter = new ListFilter(null); if (!CommonUtil.nullOrEmpty(entityId)) { filter.addQueryParam("entityId", entityId.toString()); } @@ -170,15 +163,9 @@ public class QueryResource extends EntityResource { description = "Fields requested in the returned resource", schema = @Schema(type = "string", example = FIELDS)) @QueryParam("fields") - String fieldsParam, - @Parameter( - description = "Include all, deleted, or non-deleted entities.", - schema = @Schema(implementation = Include.class)) - @QueryParam("include") - @DefaultValue("non-deleted") - Include include) + String fieldsParam) throws IOException { - return getInternal(uriInfo, securityContext, id, fieldsParam, include); + return getInternal(uriInfo, securityContext, id, fieldsParam, null); } @GET @@ -203,15 +190,9 @@ public class QueryResource extends EntityResource { description = "Fields requested in the returned resource", schema = @Schema(type = "string", example = FIELDS)) @QueryParam("fields") - String fieldsParam, - @Parameter( - description = "Include all, deleted, or non-deleted entities.", - schema = @Schema(implementation = Include.class)) - @QueryParam("include") - @DefaultValue("non-deleted") - Include include) + String fieldsParam) throws IOException { - return getByNameInternal(uriInfo, securityContext, fqn, fieldsParam, include); + return getByNameInternal(uriInfo, securityContext, fqn, fieldsParam, null); } @GET diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java index 08352144d7f..2966a1f5587 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java @@ -31,6 +31,7 @@ import java.util.UUID; import java.util.function.BiPredicate; import java.util.regex.Pattern; import java.util.stream.Collectors; +import javax.validation.constraints.NotNull; import javax.ws.rs.WebApplicationException; import lombok.Getter; import lombok.NonNull; @@ -294,6 +295,16 @@ public final class EntityUtil { return ids; } + public static boolean isDescriptionRequired(Class clz) { + // Returns true if description field in entity is required + try { + java.lang.reflect.Field description = clz.getDeclaredField(Entity.FIELD_DESCRIPTION); + return description.getAnnotation(NotNull.class) != null; + } catch (NoSuchFieldException e) { + return false; + } + } + public static class Fields { public static final Fields EMPTY_FIELDS = new Fields(Collections.emptySet()); @Getter private final Set fieldList; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index 57d659ed63e..0123f4a6665 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -220,9 +220,9 @@ public abstract class EntityResourceTest allowedFields = Entity.getEntityFields(entityClass); + this.supportsEmptyDescription = !EntityUtil.isDescriptionRequired(entityClass); this.supportsFollowers = allowedFields.contains(FIELD_FOLLOWERS); this.supportsOwner = allowedFields.contains(FIELD_OWNER); this.supportsTags = allowedFields.contains(FIELD_TAGS); @@ -566,6 +567,7 @@ public abstract class EntityResourceTest createdUUIDs = new ArrayList<>(); for (int i = 0; i < maxEntities; i++) { createdUUIDs.add(createEntity(createRequest(test, i + 1), ADMIN_AUTH_HEADERS).getId()); @@ -578,6 +580,8 @@ public abstract class EntityResourceTest allEntities = listEntities(queryParams, 1000000, null, null, ADMIN_AUTH_HEADERS); int totalRecords = allEntities.getData().size(); printEntities(allEntities); + System.out.println("XXX totalRecords " + totalRecords); // List entity with "limit" set from 1 to maxEntities size with random jumps (to reduce the test time) // Each time compare the returned list with allTables list to make sure right results are returned @@ -600,8 +605,8 @@ public abstract class EntityResourceTest backwardPage; boolean foundDeleted = false; do { // For each limit (or page size) - forward scroll till the end - LOG.debug( - "Limit {} forward pageCount {} indexInAllTables {} totalRecords {} afterCursor {}", + LOG.info( + "XXX Limit {} forward pageCount {} indexInAllTables {} totalRecords {} afterCursor {}", limit, pageCount, indexInAllTables, @@ -2489,8 +2494,8 @@ public abstract class EntityResourceTest list) { - list.getData().forEach(e -> LOG.debug("{} {}", entityClass, e.getFullyQualifiedName())); - LOG.debug("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter()); + list.getData().forEach(e -> LOG.info("XXX {} {}", entityClass, e.getFullyQualifiedName())); + LOG.info("XXX before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter()); } public void assertEntityDeleted(T entity, boolean hardDelete) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java index 12e8dd44ff4..b4979e7945f 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java @@ -22,8 +22,6 @@ import org.openmetadata.service.util.JsonUtils; public class DataProductResourceTest extends EntityResourceTest { public DataProductResourceTest() { super(Entity.DATA_PRODUCT, DataProduct.class, DataProductList.class, "dataProducts", DataProductResource.FIELDS); - supportsFieldsQueryParam = false; // TODO - supportsEmptyDescription = false; } public void setupDataProducts(TestInfo test) throws HttpResponseException { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java index e7e343d6746..88c9dc13d3f 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java @@ -24,8 +24,6 @@ import org.openmetadata.service.util.JsonUtils; public class DomainResourceTest extends EntityResourceTest { public DomainResourceTest() { super(Entity.DOMAIN, Domain.class, DomainList.class, "domains", DomainResource.FIELDS); - supportsFieldsQueryParam = false; // TODO - supportsEmptyDescription = false; } public void setupDomains(TestInfo test) throws IOException { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestDefinitionResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestDefinitionResourceTest.java index 7f6985fd5c3..a1672a12c21 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestDefinitionResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestDefinitionResourceTest.java @@ -28,7 +28,6 @@ public class TestDefinitionResourceTest extends EntityResourceTest callbackEvents = details.getEvents(); + assertNotNull(callbackEvents); assertNotNull(callbackEvents.peek()); waitAndCheckForEvents("*", "*", "*", callbackEvents.peek().getTimestamp(), callbackEvents, 30); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index fca009b24e4..5a4632c0466 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -84,9 +84,7 @@ import org.openmetadata.service.util.TestUtils.UpdateType; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class GlossaryResourceTest extends EntityResourceTest { public GlossaryResourceTest() { - // TODO add system glossary super(Entity.GLOSSARY, Glossary.class, GlossaryResource.GlossaryList.class, "glossaries", GlossaryResource.FIELDS); - supportsEmptyDescription = false; supportsSearchIndex = true; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java index 31d9dfba4cc..72744269780 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java @@ -95,7 +95,6 @@ public class GlossaryTermResourceTest extends EntityResourceTest { public KpiResourceTest() { super(Entity.KPI, Kpi.class, KpiResource.KpiList.class, "kpi", KpiResource.FIELDS); - supportsEmptyDescription = false; supportsPatch = false; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java index 0ccf015c4eb..675a6e707e3 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java @@ -57,7 +57,6 @@ public class TypeResourceTest extends EntityResourceTest { public TypeResourceTest() { super(Entity.TYPE, Type.class, TypeList.class, "metadata/types", TypeResource.PROPERTIES); - supportsEmptyDescription = false; supportsFieldsQueryParam = false; supportedNameCharacters = "_" + RANDOM_STRING_GENERATOR.generate(1); // No other special characters allowed } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/query/QueryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/query/QueryResourceTest.java index a12ce9e5bfb..b9553900bcf 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/query/QueryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/query/QueryResourceTest.java @@ -48,7 +48,6 @@ public class QueryResourceTest extends EntityResourceTest { public QueryResourceTest() { super(Entity.QUERY, Query.class, QueryResource.QueryList.class, "queries", QueryResource.FIELDS); - supportsSoftDelete = false; supportsSearchIndex = true; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/ClassificationResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/ClassificationResourceTest.java index 3d899d5e144..32a8602ff44 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/ClassificationResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/ClassificationResourceTest.java @@ -58,7 +58,6 @@ public class ClassificationResourceTest extends EntityResourceTest { public TagResourceTest() { super(Entity.TAG, Tag.class, TagList.class, "tags", TagResource.FIELDS); - supportsEmptyDescription = false; supportsSearchIndex = true; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/EntityUtilTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/EntityUtilTest.java new file mode 100644 index 00000000000..b31a219c45c --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/EntityUtilTest.java @@ -0,0 +1,15 @@ +package org.openmetadata.service.util; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.entity.data.GlossaryTerm; +import org.openmetadata.schema.entity.data.Table; + +class EntityUtilTest { + @Test + void test_isDescriptionRequired() { + assertFalse(EntityUtil.isDescriptionRequired(Table.class)); // Table entity does not require description + assertTrue(EntityUtil.isDescriptionRequired(GlossaryTerm.class)); // GlossaryTerm entity requires description + } +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/query.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/query.json index e9022f39b2f..c4168315e8f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/query.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/query.json @@ -94,11 +94,6 @@ "queryUsedIn": { "description": "Entities that are using this query", "$ref": "../../type/entityReferenceList.json" - }, - "deleted": { - "description": "When `true` indicates the entity has been soft deleted.", - "type": "boolean", - "default": false } }, "required": ["name","query"], diff --git a/openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json b/openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json index 70c40d561c5..b645823be8f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json +++ b/openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json @@ -241,11 +241,6 @@ "statusDetails": { "$ref": "#/definitions/subscriptionStatus" }, - "deleted": { - "description": "When `true` indicates the entity has been soft deleted.", - "type": "boolean", - "default": false - }, "provider" : { "$ref": "../type/basic.json#/definitions/providerType" }