yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-07 21:16:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initialize EntityResoruceTest support flags automatically (#12513)

Browse Source
This commit is contained in:
Suresh Srinivas 2023-07-27 20:42:10 -07:00 committed by GitHub
parent 02ec8b0161
commit 8830d1707d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 76 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql
View File

@ -60,3 +60,7 @@ SET json = JSON_REPLACE(
)
where de2.serviceType = 'Mssql'
and JSON_EXTRACT(json, '$.connection.config.database') is NULL;
-- column deleted not needed for entities that don't support soft delete
ALTER TABLE query_entity DROP COLUMN deleted;
ALTER TABLE event_subscription_entity DROP COLUMN deleted;

6
bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql
View File

@ -37,7 +37,7 @@ where json#>>'{connection,config,scheme}' in ('impala', 'impala4');
-- remove the dataModel references from Data Models
UPDATE dashboard_data_model_entity SET json = json #- '{dataModels}';
-- migrate ingestAllDatabases in mssql
-- migrate ingestAllDatabases in mssql
UPDATE dbservice_entity de2
SET json = JSONB_SET(
json || JSONB_SET(json,'{connection,config}', json#>'{connection,config}'||
@ -56,3 +56,7 @@ SET json = JSONB_SET(
)
WHERE de2.serviceType = 'Mssql'
AND json->>'{connection,config,database}' IS NULL;
-- column deleted not needed for entities that don't support soft delete
ALTER TABLE query_entity DROP COLUMN deleted;
ALTER TABLE event_subscription_entity DROP COLUMN deleted;

19
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -1424,6 +1424,11 @@ public interface CollectionDAO {
@SqlQuery("SELECT json FROM <table>")
List<String> listAllEventsSubscriptions(@Define("table") String table);
@Override
default boolean supportsSoftDelete() {
return false;
}
}
interface ChartDAO extends EntityDAO<Chart> {
@ -1800,6 +1805,11 @@ public interface CollectionDAO {
return "nameHash";
}
@Override
default boolean supportsSoftDelete() {
return false;
}
@Override
default int listCount(ListFilter filter) {
String entityId = filter.getQueryParam("entityId");
@ -3267,15 +3277,6 @@ public interface CollectionDAO {
+ "ORDER BY timestamp DESC LIMIT 1")
String getLatestExtensionByFQN(@Bind("entityFQNHash") String entityFQNHash, @Bind("jsonSchema") String jsonSchema);
@SqlQuery(
"SELECT json FROM entity_extension_time_series where entityFQNHash = :entityFQNHash and jsonSchema = :jsonSchema "
+ " AND timestamp >= :startTs and timestamp <= :endTs ORDER BY timestamp DESC")
List<String> listBetweenTimestampsByFQN(
@Bind("entityFQNHash") String entityFQNHash,
@Bind("jsonSchema") String jsonSchema,
@Bind("startTs") Long startTs,
@Bind("endTs") long endTs);
@SqlQuery(
"SELECT json FROM entity_extension_time_series where entityFQNHash = :entityFQNHash and extension = :extension "
+ " AND timestamp >= :startTs and timestamp <= :endTs ORDER BY timestamp DESC")

3
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java
View File

@ -15,7 +15,6 @@ package org.openmetadata.service.jdbi3;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.service.Entity.DOMAIN;
import static org.openmetadata.service.Entity.GLOSSARY_TERM;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
@ -59,7 +58,7 @@ public class DomainRepository extends EntityRepository<Domain> {
private List<EntityReference> getChildren(Domain entity) throws IOException {
List<EntityRelationshipRecord> ids = findTo(entity.getId(), DOMAIN, Relationship.CONTAINS, DOMAIN);
return EntityUtil.populateEntityReferences(ids, GLOSSARY_TERM);
return EntityUtil.populateEntityReferences(ids, DOMAIN);
}
private List<EntityReference> getExperts(Domain entity) throws IOException {

12
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
View File

@ -1712,21 +1712,21 @@ public abstract class EntityRepository<T extends EntityInterface> {
// Check for updated and deleted fields
for (Iterator<Entry<String, JsonNode>> it = origFields.fields(); it.hasNext(); ) {
Entry<String, JsonNode> orig = it.next();
JsonNode updated = updatedFields.get(orig.getKey());
if (updated == null) {
JsonNode updatedField = updatedFields.get(orig.getKey());
if (updatedField == null) {
deleted.add(JsonUtils.getObjectNode(orig.getKey(), orig.getValue()));
} else {
// TODO converting to a string is a hack for now because JsonNode equals issues
recordChange(getExtensionField(orig.getKey()), orig.getValue().toString(), updated.toString());
recordChange(getExtensionField(orig.getKey()), orig.getValue().toString(), updatedField.toString());
}
}
// Check for added fields
for (Iterator<Entry<String, JsonNode>> it = updatedFields.fields(); it.hasNext(); ) {
Entry<String, JsonNode> updated = it.next();
JsonNode orig = origFields.get(updated.getKey());
Entry<String, JsonNode> updatedField = it.next();
JsonNode orig = origFields.get(updatedField.getKey());
if (orig == null) {
added.add(JsonUtils.getObjectNode(updated.getKey(), updated.getValue()));
added.add(JsonUtils.getObjectNode(updatedField.getKey(), updatedField.getValue()));
}
}
if (!added.isEmpty()) {

31
openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java
View File

@ -58,7 +58,6 @@ import org.openmetadata.schema.entity.events.EventSubscription;
import org.openmetadata.schema.entity.events.SubscriptionStatus;
import org.openmetadata.schema.type.EntityHistory;
import org.openmetadata.schema.type.Function;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.schema.type.SubscriptionResourceDescriptor;
import org.openmetadata.service.Entity;
@ -192,15 +191,9 @@ public class EventSubscriptionResource extends EntityResource<EventSubscription,
description = "Returns list of event subscriptions after this cursor",
schema = @Schema(type = "string"))
@QueryParam("after")
String after,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
String after)
throws IOException {
ListFilter filter = new ListFilter(include);
ListFilter filter = new ListFilter(null);
return listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after);
}
@ -228,15 +221,9 @@ public class EventSubscriptionResource extends EntityResource<EventSubscription,
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
String fieldsParam)
throws IOException {
return getInternal(uriInfo, securityContext, id, fieldsParam, include);
return getInternal(uriInfo, securityContext, id, fieldsParam, null);
}
@GET
@ -265,15 +252,9 @@ public class EventSubscriptionResource extends EntityResource<EventSubscription,
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
String fieldsParam)
throws IOException {
return getByNameInternal(uriInfo, securityContext, name, fieldsParam, include);
return getByNameInternal(uriInfo, securityContext, name, fieldsParam, null);
}
@POST

31
openmetadata-service/src/main/java/org/openmetadata/service/resources/query/QueryResource.java
View File

@ -43,7 +43,6 @@ import org.openmetadata.schema.entity.data.Query;
import org.openmetadata.schema.type.ChangeEvent;
import org.openmetadata.schema.type.EntityHistory;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.schema.type.Votes;
import org.openmetadata.schema.utils.EntityInterfaceUtil;
@ -132,15 +131,9 @@ public class QueryResource extends EntityResource<Query, QueryRepository> {
String before,
@Parameter(description = "Returns list of queries after this cursor", schema = @Schema(type = "string"))
@QueryParam("after")
String after,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
String after)
throws IOException {
ListFilter filter = new ListFilter(include);
ListFilter filter = new ListFilter(null);
if (!CommonUtil.nullOrEmpty(entityId)) {
filter.addQueryParam("entityId", entityId.toString());
}
@ -170,15 +163,9 @@ public class QueryResource extends EntityResource<Query, QueryRepository> {
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
String fieldsParam)
throws IOException {
return getInternal(uriInfo, securityContext, id, fieldsParam, include);
return getInternal(uriInfo, securityContext, id, fieldsParam, null);
}
@GET
@ -203,15 +190,9 @@ public class QueryResource extends EntityResource<Query, QueryRepository> {
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
String fieldsParam)
throws IOException {
return getByNameInternal(uriInfo, securityContext, fqn, fieldsParam, include);
return getByNameInternal(uriInfo, securityContext, fqn, fieldsParam, null);
}
@GET

11
openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java
View File

@ -31,6 +31,7 @@ import java.util.UUID;
import java.util.function.BiPredicate;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.validation.constraints.NotNull;
import javax.ws.rs.WebApplicationException;
import lombok.Getter;
import lombok.NonNull;
@ -294,6 +295,16 @@ public final class EntityUtil {
return ids;
}
public static <T> boolean isDescriptionRequired(Class<T> clz) {
// Returns true if description field in entity is required
try {
java.lang.reflect.Field description = clz.getDeclaredField(Entity.FIELD_DESCRIPTION);
return description.getAnnotation(NotNull.class) != null;
} catch (NoSuchFieldException e) {
return false;
}
}
public static class Fields {
public static final Fields EMPTY_FIELDS = new Fields(Collections.emptySet());
@Getter private final Set<String> fieldList;

17
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -220,9 +220,9 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
protected final boolean supportsOwner;
protected final boolean supportsTags;
protected boolean supportsPatch = true;
protected boolean supportsSoftDelete;
protected final boolean supportsSoftDelete;
protected boolean supportsFieldsQueryParam = true;
protected boolean supportsEmptyDescription = true;
protected final boolean supportsEmptyDescription;
// Special characters supported in the entity name
protected String supportedNameCharacters = "_'-.&()" + RANDOM_STRING_GENERATOR.generate(1);
@ -382,6 +382,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
this.allFields = fields;
ENTITY_RESOURCE_TEST_MAP.put(entityType, this);
Set<String> allowedFields = Entity.getEntityFields(entityClass);
this.supportsEmptyDescription = !EntityUtil.isDescriptionRequired(entityClass);
this.supportsFollowers = allowedFields.contains(FIELD_FOLLOWERS);
this.supportsOwner = allowedFields.contains(FIELD_OWNER);
this.supportsTags = allowedFields.contains(FIELD_TAGS);
@ -566,6 +567,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
Random rand = new Random();
int maxEntities = rand.nextInt(16) + 5;
System.out.println("XXX creating entities " + maxEntities);
List<UUID> createdUUIDs = new ArrayList<>();
for (int i = 0; i < maxEntities; i++) {
createdUUIDs.add(createEntity(createRequest(test, i + 1), ADMIN_AUTH_HEADERS).getId());
@ -578,6 +580,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
// Test listing entities that include deleted, non-deleted, and all the entities
Random random = new Random();
for (Include include : List.of(Include.NON_DELETED, Include.ALL, Include.DELETED)) {
System.out.println("XXX supportSoftDelete " + supportsSoftDelete);
System.out.println("XXX Include " + include);
if (!supportsSoftDelete && include.equals(Include.DELETED)) {
continue;
}
@ -588,6 +592,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
ResultList<T> allEntities = listEntities(queryParams, 1000000, null, null, ADMIN_AUTH_HEADERS);
int totalRecords = allEntities.getData().size();
printEntities(allEntities);
System.out.println("XXX totalRecords " + totalRecords);
// List entity with "limit" set from 1 to maxEntities size with random jumps (to reduce the test time)
// Each time compare the returned list with allTables list to make sure right results are returned
@ -600,8 +605,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
ResultList<T> backwardPage;
boolean foundDeleted = false;
do { // For each limit (or page size) - forward scroll till the end
LOG.debug(
"Limit {} forward pageCount {} indexInAllTables {} totalRecords {} afterCursor {}",
LOG.info(
"XXX Limit {} forward pageCount {} indexInAllTables {} totalRecords {} afterCursor {}",
limit,
pageCount,
indexInAllTables,
@ -2489,8 +2494,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
}
private void printEntities(ResultList<T> list) {
list.getData().forEach(e -> LOG.debug("{} {}", entityClass, e.getFullyQualifiedName()));
LOG.debug("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
list.getData().forEach(e -> LOG.info("XXX {} {}", entityClass, e.getFullyQualifiedName()));
LOG.info("XXX before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
}
public void assertEntityDeleted(T entity, boolean hardDelete) {

2
openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java
View File

@ -22,8 +22,6 @@ import org.openmetadata.service.util.JsonUtils;
public class DataProductResourceTest extends EntityResourceTest<DataProduct, CreateDataProduct> {
public DataProductResourceTest() {
super(Entity.DATA_PRODUCT, DataProduct.class, DataProductList.class, "dataProducts", DataProductResource.FIELDS);
supportsFieldsQueryParam = false; // TODO
supportsEmptyDescription = false;
}
public void setupDataProducts(TestInfo test) throws HttpResponseException {

2
openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java
View File

@ -24,8 +24,6 @@ import org.openmetadata.service.util.JsonUtils;
public class DomainResourceTest extends EntityResourceTest<Domain, CreateDomain> {
public DomainResourceTest() {
super(Entity.DOMAIN, Domain.class, DomainList.class, "domains", DomainResource.FIELDS);
supportsFieldsQueryParam = false; // TODO
supportsEmptyDescription = false;
}
public void setupDomains(TestInfo test) throws IOException {

1
openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestDefinitionResourceTest.java
View File

@ -28,7 +28,6 @@ public class TestDefinitionResourceTest extends EntityResourceTest<TestDefinitio
TestDefinitionResource.TestDefinitionList.class,
"dataQuality/testDefinitions",
TestDefinitionResource.FIELDS);
supportsEmptyDescription = false;
}
public void setupTestDefinitions() throws IOException {

2
openmetadata-service/src/test/java/org/openmetadata/service/resources/events/EventSubscriptionResourceTest.java
View File

@ -57,7 +57,6 @@ public class EventSubscriptionResourceTest extends EntityResourceTest<EventSubsc
"events/subscriptions",
EventSubscriptionResource.FIELDS);
supportedNameCharacters = supportedNameCharacters.replace(" ", ""); // Space not supported
supportsSoftDelete = false;
supportsFieldsQueryParam = false;
}
@ -323,6 +322,7 @@ public class EventSubscriptionResourceTest extends EntityResourceTest<EventSubsc
// Now check state of webhooks created
WebhookCallbackResource.EventDetails details = waitForFirstEvent("simulate-slowServer", 25);
ConcurrentLinkedQueue<ChangeEvent> callbackEvents = details.getEvents();
assertNotNull(callbackEvents);
assertNotNull(callbackEvents.peek());
waitAndCheckForEvents("*", "*", "*", callbackEvents.peek().getTimestamp(), callbackEvents, 30);

2
openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java
View File

@ -84,9 +84,7 @@ import org.openmetadata.service.util.TestUtils.UpdateType;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlossary> {
public GlossaryResourceTest() {
// TODO add system glossary
super(Entity.GLOSSARY, Glossary.class, GlossaryResource.GlossaryList.class, "glossaries", GlossaryResource.FIELDS);
supportsEmptyDescription = false;
supportsSearchIndex = true;
}

1
openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java
View File

@ -95,7 +95,6 @@ public class GlossaryTermResourceTest extends EntityResourceTest<GlossaryTerm, C
GlossaryTermResource.GlossaryTermList.class,
"glossaryTerms",
GlossaryTermResource.FIELDS);
supportsEmptyDescription = false;
}
@Order(0)

1
openmetadata-service/src/test/java/org/openmetadata/service/resources/kpi/KpiResourceTest.java
View File

@ -48,7 +48,6 @@ import org.openmetadata.service.util.TestUtils;
public class KpiResourceTest extends EntityResourceTest<Kpi, CreateKpiRequest> {
public KpiResourceTest() {
super(Entity.KPI, Kpi.class, KpiResource.KpiList.class, "kpi", KpiResource.FIELDS);
supportsEmptyDescription = false;
supportsPatch = false;
}

1
openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java
View File

@ -57,7 +57,6 @@ public class TypeResourceTest extends EntityResourceTest<Type, CreateType> {
public TypeResourceTest() {
super(Entity.TYPE, Type.class, TypeList.class, "metadata/types", TypeResource.PROPERTIES);
supportsEmptyDescription = false;
supportsFieldsQueryParam = false;
supportedNameCharacters = "_" + RANDOM_STRING_GENERATOR.generate(1); // No other special characters allowed
}

1
openmetadata-service/src/test/java/org/openmetadata/service/resources/query/QueryResourceTest.java
View File

@ -48,7 +48,6 @@ public class QueryResourceTest extends EntityResourceTest<Query, CreateQuery> {
public QueryResourceTest() {
super(Entity.QUERY, Query.class, QueryResource.QueryList.class, "queries", QueryResource.FIELDS);
supportsSoftDelete = false;
supportsSearchIndex = true;
}

1
openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/ClassificationResourceTest.java
View File

@ -58,7 +58,6 @@ public class ClassificationResourceTest extends EntityResourceTest<Classificatio
ClassificationList.class,
"classifications",
ClassificationResource.FIELDS);
supportsEmptyDescription = false;
}
@Test

1
openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/TagResourceTest.java
View File

@ -68,7 +68,6 @@ public class TagResourceTest extends EntityResourceTest<Tag, CreateTag> {
public TagResourceTest() {
super(Entity.TAG, Tag.class, TagList.class, "tags", TagResource.FIELDS);
supportsEmptyDescription = false;
supportsSearchIndex = true;
}

15
openmetadata-service/src/test/java/org/openmetadata/service/util/EntityUtilTest.java Normal file
View File

@ -0,0 +1,15 @@
package org.openmetadata.service.util;
import static org.junit.jupiter.api.Assertions.*;
import org.junit.jupiter.api.Test;
import org.openmetadata.schema.entity.data.GlossaryTerm;
import org.openmetadata.schema.entity.data.Table;
class EntityUtilTest {
@Test
void test_isDescriptionRequired() {
assertFalse(EntityUtil.isDescriptionRequired(Table.class)); // Table entity does not require description
assertTrue(EntityUtil.isDescriptionRequired(GlossaryTerm.class)); // GlossaryTerm entity requires description
}
}

5
openmetadata-spec/src/main/resources/json/schema/entity/data/query.json
View File

@ -94,11 +94,6 @@
"queryUsedIn": {
"description": "Entities that are using this query",
"$ref": "../../type/entityReferenceList.json"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
}
},
"required": ["name","query"],

5
openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json
View File

@ -241,11 +241,6 @@
"statusDetails": {
"$ref": "#/definitions/subscriptionStatus"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
},
"provider" : {
"$ref": "../type/basic.json#/definitions/providerType"
}