diff --git a/bootstrap/sql/mysql/v001__create_db_connection_info.sql b/bootstrap/sql/mysql/v001__create_db_connection_info.sql index 27a724382da..d5241b80429 100644 --- a/bootstrap/sql/mysql/v001__create_db_connection_info.sql +++ b/bootstrap/sql/mysql/v001__create_db_connection_info.sql @@ -395,8 +395,8 @@ CREATE TABLE IF NOT EXISTS change_event ( entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityType') NOT NULL, userName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.userName') NOT NULL, dateTime TIMESTAMP GENERATED ALWAYS AS (TIMESTAMP(STR_TO_DATE(json ->> '$.dateTime', '%Y-%m-%dT%T.%fZ'))) NOT NULL, +-- dateTime DATE GENERATED ALWAYS AS (STR_TO_DATE(json ->> '$.dateTime', '%Y-%m-%dT%T.%fZ')) NOT NULL, json JSON NOT NULL, - timestamp BIGINT, INDEX (dateTime), INDEX (eventType), INDEX (entityType) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java index 56889e66e84..b1b4b1a8ea7 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java @@ -82,6 +82,15 @@ public class CatalogApplication extends Application { final JdbiFactory factory = new JdbiFactory(); final Jdbi jdbi = factory.build(environment, catalogConfig.getDataSourceFactory(), "mysql3"); + SqlLogger sqlLogger = new SqlLogger() { + @Override + public void logAfterExecution(StatementContext context) { + LOG.info("sql {}, parameters {}, timeTaken {} ms", context.getRenderedSql(), + context.getBinding().toString(), context.getElapsedTime(ChronoUnit.MILLIS)); + } + }; + jdbi.setSqlLogger(sqlLogger); + // Register Authorizer registerAuthorizer(catalogConfig, environment, jdbi); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java index e4e6df668bb..8bbd8306600 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java @@ -26,6 +26,7 @@ import org.openmetadata.catalog.util.EntityInterface; import javax.ws.rs.core.UriInfo; import java.io.IOException; import java.net.URI; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -139,5 +140,25 @@ public final class Entity { } return entityRepository.getEntityInterface(entity); } + + public static class EntityList { + public static final EntityList EMPTY_LIST = new EntityList(null); + private final List list; + + public EntityList(String entitiesParam) { + if (entitiesParam == null) { + list = Collections.emptyList(); + return; + } + list = Arrays.asList(entitiesParam.replaceAll("\\s", "").split(",")); + for (String field : list) { + // TODO validate entity + } + } + + public List getList() { + return list; + } + } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/events/ChangeEventHandler.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/events/ChangeEventHandler.java index d5bb6701314..5d1bda88987 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/events/ChangeEventHandler.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/events/ChangeEventHandler.java @@ -21,8 +21,8 @@ import org.openmetadata.catalog.CatalogApplicationConfig; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.jdbi3.CollectionDAO; import org.openmetadata.catalog.type.ChangeEvent; -import org.openmetadata.catalog.type.ChangeEvent.EventType; import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.type.EventType; import org.openmetadata.catalog.util.EntityInterface; import org.openmetadata.catalog.util.JsonUtils; import org.openmetadata.catalog.util.RestUtil; @@ -54,11 +54,11 @@ public class ChangeEventHandler implements EventHandler { try { Object entity = responseContext.getEntity(); String changeType = responseContext.getHeaderString(RestUtil.CHANGE_CUSTOM_HEADER); - System.out.println("Change type is " + changeType); if (responseCode == Status.CREATED.getStatusCode()) { EntityInterface entityInterface = Entity.getEntityInterface(entity); EntityReference entityReference = Entity.getEntityReference(entity); + System.out.println("Entity created at " + entityInterface.getUpdatedAt().getTime()); changeEvent = new ChangeEvent() .withEventType(EventType.ENTITY_CREATED) .withEntityId(entityInterface.getId()) @@ -72,10 +72,7 @@ public class ChangeEventHandler implements EventHandler { } else if (changeType.equals(RestUtil.ENTITY_UPDATED)) { EntityInterface entityInterface = Entity.getEntityInterface(entity); EntityReference entityReference = Entity.getEntityReference(entity); - - System.out.println(entityInterface.getId()); - System.out.println(entity); - System.out.println(entityReference.getType()); + System.out.println("Entity updated at " + entityInterface.getUpdatedAt().getTime()); changeEvent = new ChangeEvent() .withEventType(EventType.ENTITY_UPDATED) .withEntityId(entityInterface.getId()) @@ -83,7 +80,7 @@ public class ChangeEventHandler implements EventHandler { .withUserName(entityInterface.getUpdatedBy()) .withDateTime(entityInterface.getUpdatedAt()) .withChangeDescription(entityInterface.getChangeDescription()) - .withPreviousVersion(entityInterface.getVersion()) + .withPreviousVersion(entityInterface.getChangeDescription().getPreviousVersion()) .withCurrentVersion(entityInterface.getVersion()); } else if (changeType.equals(RestUtil.ENTITY_FIELDS_CHANGED)){ @@ -93,8 +90,10 @@ public class ChangeEventHandler implements EventHandler { } if (changeEvent != null) { - System.out.println("Adding change " + changeEvent); dao.changeEventDAO().insert(JsonUtils.pojoToJson(changeEvent)); + System.out.println("Adding change event " + changeEvent); + } else { + System.out.println("Change event not recorded"); } } catch(Exception e) { LOG.error("Failed to capture change event for method {} due to {}", method, e); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java index e97d35efed6..8c114d02b20 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java @@ -17,16 +17,22 @@ package org.openmetadata.catalog.jdbi3; import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.catalog.Entity.EntityList; import org.openmetadata.catalog.resources.events.EventResource.ChangeEventList; import org.openmetadata.catalog.type.ChangeEvent; +import org.openmetadata.catalog.type.EventType; import org.openmetadata.catalog.util.JsonUtils; +import org.openmetadata.catalog.util.RestUtil; import org.openmetadata.catalog.util.ResultList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.security.GeneralSecurityException; +import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Comparator; +import java.util.Date; import java.util.List; public class ChangeEventRepository { @@ -36,14 +42,27 @@ public class ChangeEventRepository { public ChangeEventRepository(CollectionDAO dao) { this.dao = dao; } @Transaction - public ResultList list(String date, List eventTypes, List entityTypes) throws IOException, + public ResultList list(Date date, EntityList entityCreatedList, + EntityList entityUpdatedList, EntityList entityDeletedList) throws IOException, GeneralSecurityException { - List jsons = dao.changeEventDAO().list(eventTypes, entityTypes, date); + String dateStr = RestUtil.DATE_TIME_FORMAT.format(date); + Timestamp timestamp = new Timestamp(date.getTime()); + System.out.println("Listing from timestamp " + timestamp); + System.out.println("Listing from timestamp " + timestamp.getTime()); + System.out.println("Listing from time " + date.getTime()); + List jsons = new ArrayList<>(); + jsons.addAll(dao.changeEventDAO().list(EventType.ENTITY_CREATED.value(), entityCreatedList.getList(), + timestamp.getTime())); + jsons.addAll(dao.changeEventDAO().list(EventType.ENTITY_UPDATED.value(), entityUpdatedList.getList(), + timestamp.getTime())); + jsons.addAll(dao.changeEventDAO().list(EventType.ENTITY_DELETED.value(), entityDeletedList.getList(), + timestamp.getTime())); System.out.println("Total change events " + jsons.size()); List changeEvents = new ArrayList<>(); for (String json : jsons) { changeEvents.add(JsonUtils.readValue(json, ChangeEvent.class)); } + changeEvents.sort(Comparator.comparing((ChangeEvent changeEvent) -> changeEvent.getDateTime().getTime()).reversed()); return new ChangeEventList(changeEvents, null, null, changeEvents.size()); // TODO } @@ -52,4 +71,5 @@ public class ChangeEventRepository { dao.changeEventDAO().insert(JsonUtils.pojoToJson(changeEvent)); return changeEvent; } + } \ No newline at end of file diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java index 392b24d3e09..7651570c4ef 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java @@ -73,7 +73,9 @@ import org.openmetadata.catalog.util.EntityUtil; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Timestamp; import java.util.Arrays; +import java.util.Date; import java.util.List; public interface CollectionDAO { @@ -808,13 +810,13 @@ public interface CollectionDAO { void insert(@Bind("json") String json); @SqlQuery("SELECT json FROM change_event WHERE " + - "(eventType IN () OR eventType IS NULL) AND " + + "eventType = :eventType AND " + // "(entityType IN () OR entityType IS NULL) " + "(entityType IN () OR entityType IS NULL) AND " + - "dateTime >= :date " + + "dateTime >= :dateTime " + "ORDER BY dateTime DESC") - List list(@BindList("eventTypes") List eventTypes, + List list(@Bind("eventType") String eventType, @BindList("entityTypes") List entityTypes, - @Bind("date") String date); + @Bind("dateTime") long dateTime); } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java index b0ade2d5a95..3a267317988 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityRepository.java @@ -200,7 +200,6 @@ public abstract class EntityRepository { @Transaction public final PutResponse createOrUpdate(UriInfo uriInfo, T updated) throws IOException, ParseException { - String change; validate(updated); T original = JsonUtils.readValue(dao.findJsonByFqn(getFullyQualifiedName(updated)), entityClass); if (original == null) { @@ -212,7 +211,7 @@ public abstract class EntityRepository { EntityUpdater entityUpdater = getUpdater(original, updated, false); entityUpdater.update(); entityUpdater.store(); - change = entityUpdater.fieldsChanged() ? RestUtil.ENTITY_UPDATED : RestUtil.ENTITY_NO_CHANGE; + String change = entityUpdater.fieldsChanged() ? RestUtil.ENTITY_UPDATED : RestUtil.ENTITY_NO_CHANGE; return new PutResponse<>(Status.OK, withHref(uriInfo, updated), change); } @@ -485,7 +484,11 @@ public abstract class EntityRepository { JsonUtils.pojoToJson(original.getEntity())); // Store the new version + System.out.println("Storing new version"); EntityRepository.this.store(updated.getEntity(), true); + } else { + System.out.println("Restoring old version"); + updated.setUpdateDetails(original.getUpdatedBy(), original.getUpdatedAt()); } } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java index 573d4c7ad23..3a6848aa7cd 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java @@ -23,13 +23,12 @@ import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; -import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.Entity.EntityList; import org.openmetadata.catalog.jdbi3.ChangeEventRepository; import org.openmetadata.catalog.jdbi3.CollectionDAO; import org.openmetadata.catalog.resources.Collection; import org.openmetadata.catalog.security.CatalogAuthorizer; import org.openmetadata.catalog.type.ChangeEvent; -import org.openmetadata.catalog.type.ChangeEvent.EventType; import org.openmetadata.catalog.util.RestUtil; import org.openmetadata.catalog.util.ResultList; @@ -84,25 +83,26 @@ public class EventResource { schema = @Schema(implementation = ChangeEvent.class))), @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") }) - public ResultList get( - @Context UriInfo uriInfo, - @Parameter(description = "Entity type for which usage is requested", - required = true, - schema = @Schema(type = "string", example = "table, report, metrics, or dashboard")) - @QueryParam("entityTypes") List entityTypes, - @Parameter(description = "Event types", - required = true, - schema = @Schema(type = "string")) - @QueryParam("eventTypes") List eventTypes, - @Parameter(description = "Events since this date and time (inclusive) in ISO 8601 format.", - required = true, - schema = @Schema(type = "string")) - @QueryParam("date") String date) throws IOException, GeneralSecurityException, ParseException { - // TODO hack - eventTypes = List.of(EventType.ENTITY_CREATED.toString(), EventType.ENTITY_UPDATED.toString()); - entityTypes = List.of(Entity.TABLE); + public ResultList get(@Context UriInfo uriInfo, + // TODO document + @Parameter(description = "Entities requested for `entityCreated` event", + schema = @Schema(type = "string", example = "table,dashboard,...")) + @QueryParam("entityCreated") String entityCreated, + @Parameter(description = "Entities requested for `entityUpdated` event", + schema = @Schema(type = "string", example = "table,dashboard,...")) + @QueryParam("entityUpdated") String entityUpdated, + @Parameter(description = "Entities requested for `entityDeleted` event", + schema = @Schema(type = "string", example = "table,dashboard,...")) + @QueryParam("entityDeleted") String entityDeleted, + @Parameter(description = "Events starting from this date time in ISO8601 format", + required = true, + schema = @Schema(type = "string", example = "2021-01-28T10:00:00.000000Z")) + @QueryParam("date") String date) + throws IOException, GeneralSecurityException, ParseException { Date parsedDate = RestUtil.DATE_TIME_FORMAT.parse(date); - date = RestUtil.DATE_FORMAT.format(parsedDate); - return dao.list(date, eventTypes, entityTypes); + EntityList entityCreatedList = new EntityList(entityCreated); + EntityList entityUpdatedList = new EntityList(entityCreated); + EntityList entityDeletedList = new EntityList(entityCreated); + return dao.list(parsedDate, entityCreatedList, entityUpdatedList, entityDeletedList); } } \ No newline at end of file diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/RestUtil.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/RestUtil.java index a1a670b9080..c6305452c1e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/RestUtil.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/RestUtil.java @@ -171,7 +171,6 @@ public final class RestUtil { public T getEntity() { return entity; } public Response toResponse() { - System.out.println("Status " + status + " changeType " + changeType); ResponseBuilder responseBuilder = Response.status(status).header(CHANGE_CUSTOM_HEADER, changeType); if (changeType.equals(RestUtil.ENTITY_CREATED) || changeType.equals(RestUtil.ENTITY_UPDATED) || changeType.equals(RestUtil.ENTITY_NO_CHANGE)) { diff --git a/catalog-rest-service/src/main/resources/json/schema/type/changeEvent.json b/catalog-rest-service/src/main/resources/json/schema/type/changeEvent.json index 0173fbc8cc0..1b457155684 100644 --- a/catalog-rest-service/src/main/resources/json/schema/type/changeEvent.json +++ b/catalog-rest-service/src/main/resources/json/schema/type/changeEvent.json @@ -4,9 +4,11 @@ "title": "ChangeEvent", "description": "This schema defines the change event type to capture the changes to entities. Entities change due to user activity, such as updating description of a dataset, changing ownership, or adding new tags. Entity also changes due to activities at the metadata sources, such as a new dataset was created, a datasets was deleted, or schema of a dataset is modified. When state of entity changes, an event is produced. These events can be used to build apps and bots that respond to the change from activities.", "type": "object", + "javaType": "org.openmetadata.catalog.type.ChangeEvent", "definitions": { "eventType" :{ + "javaType": "org.openmetadata.catalog.type.EventType", "description": "Type of event", "type": "string", "enum": [ @@ -15,20 +17,23 @@ "entityDeleted" ] }, - "versionChange" : { - "description": "Details of version change for `entityCreated`, `entityUpdated`, and `entityDeleted` events", - "type" : "object", + "eventFilter": { + "type": "object", + "javaType": "org.openmetadata.catalog.type.EventFilter", "properties": { - "oldVersion" : { - "$ref" : "entityHistory.json#/definitions/entityVersion" + "eventType": { + "description": "Event type that is being requested.", + "$ref": "#/definitions/eventType" }, - "newVersion" : { - "$ref" : "entityHistory.json#/definitions/entityVersion" - }, - "entity" : { - "description": "JSON coded string for the entity using the schema corresponding to `entityType` that includes the current value of entity with `changeDescription` attribute that provides details on fields added, updated, and deleted." + "entities": { + "description": "Entities for which the events are needed. Example - `table`, `topic`, etc. **When not set, events for all the entities will be provided.**", + "type": "array", + "items": { + "type": "string" + } } - } + }, + "required": ["eventType"] } }, "properties" : { diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java index d7913dc49ef..4dc1805cda7 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java @@ -29,9 +29,9 @@ import org.openmetadata.catalog.resources.teams.TeamResourceTest; import org.openmetadata.catalog.resources.teams.UserResourceTest; import org.openmetadata.catalog.type.ChangeDescription; import org.openmetadata.catalog.type.ChangeEvent; -import org.openmetadata.catalog.type.ChangeEvent.EventType; import org.openmetadata.catalog.type.EntityHistory; import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.type.EventType; import org.openmetadata.catalog.type.FieldChange; import org.openmetadata.catalog.type.TagLabel; import org.openmetadata.catalog.util.EntityInterface; @@ -83,6 +83,7 @@ import static org.openmetadata.catalog.util.TestUtils.userAuthHeaders; public abstract class EntityResourceTest extends CatalogApplicationTest { private static final Logger LOG = LoggerFactory.getLogger(EntityResourceTest.class); + private final String entityName; private final Class entityClass; private final Class> entityListClass; private final String collectionName; @@ -110,8 +111,10 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { public static final TagLabel USER_BANK_ACCOUNT_TAG_LABEL = new TagLabel().withTagFQN("User.BankAccount"); public static final TagLabel TIER1_TAG_LABEL = new TagLabel().withTagFQN("Tier.Tier1"); - public EntityResourceTest(Class entityClass, Class> entityListClass, String collectionName, + public EntityResourceTest(String entityName, Class entityClass, Class> entityListClass, + String collectionName, String fields, boolean supportsFollowers, boolean supportsOwner, boolean supportsTags) { + this.entityName = entityName; this.entityClass = entityClass; this.entityListClass = entityListClass; this.collectionName = collectionName; @@ -196,7 +199,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { throws HttpResponseException; // Entity specific validate for entity create using PATCH - public abstract void validatePatchedEntity(T expected, T updated, Map authHeaders) + public abstract void compareEntities(T expected, T updated, Map authHeaders) throws HttpResponseException; // Get interface to access all common entity attributes @@ -626,7 +629,6 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { EntityInterface entityInterface = getEntityInterface(updated); validateUpdatedEntity(updated, request, authHeaders); validateChangeDescription(updated, updateType, changeDescription); - validateEntityHistory(entityInterface.getId(), updateType, changeDescription, authHeaders); // GET ../entity/{id}/versions/{versionId} to get specific versions of the entity @@ -643,15 +645,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { T getEntity = getEntity(entityInterface.getId(), authHeaders); validateUpdatedEntity(getEntity, request, authHeaders); validateChangeDescription(getEntity, updateType, changeDescription); - - // Finally get changeEvents and ensure it is correct - List entityNames = List.of(entityInterface.getEntityReference().getType()); - List eventTypes = List.of(EventType.ENTITY_CREATED, EventType.ENTITY_UPDATED); - ResultList changeEvents = getChangeEvents(eventTypes, entityNames, entityInterface.getUpdatedAt(), - authHeaders); - System.out.println("data " + changeEvents.getData()); - System.out.println("size " + changeEvents.getData().size()); - changeEvents.getData().forEach(e -> System.out.println(e)); + validateChangeEvents(entityInterface, updateType, authHeaders); return updated; } @@ -680,12 +674,12 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { // Validate information returned in patch response has the updates T returned = patchEntity(entityInterface.getId(), originalJson, updated, authHeaders); - validatePatchedEntity(updated, returned, authHeaders); + compareEntities(updated, returned, authHeaders); validateChangeDescription(returned, updateType, expectedChange); // GET the entity and Validate information returned T getEntity = getEntity(entityInterface.getId(), authHeaders); - validatePatchedEntity(updated, getEntity, authHeaders); + compareEntities(updated, getEntity, authHeaders); validateChangeDescription(getEntity, updateType, expectedChange); return returned; } @@ -718,16 +712,45 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { } } + protected final void validateChangeEvents(EntityInterface entityInterface, UpdateType updateType, + Map authHeaders) throws IOException { + ResultList changeEvents = getChangeEvents(entityName, entityName, null, + entityInterface.getUpdatedAt(), authHeaders); + + assertTrue(changeEvents.getData().size() > 0); + + // Top most changeEvent corresponds to the update + ChangeEvent changeEvent = changeEvents.getData().get(0); + + assertEquals(changeEvent.getDateTime().getTime(), entityInterface.getUpdatedAt().getTime()); + assertEquals(changeEvent.getEntityId(), entityInterface.getId()); + assertEquals(changeEvent.getCurrentVersion(), entityInterface.getVersion()); + assertEquals(changeEvent.getUserName(), entityInterface.getUpdatedBy()); + assertEquals(changeEvent.getEntityType(), entityName); + + if (updateType == UpdateType.CREATED) { + assertEquals(changeEvent.getEventType(), EventType.ENTITY_CREATED); + assertEquals(changeEvent.getPreviousVersion(), 0.1); + assertNull(changeEvent.getChangeDescription()); + compareEntities(entityInterface.getEntity(), + JsonUtils.readValue((String)changeEvent.getEntity(), entityClass), authHeaders); + } else if (updateType == MINOR_UPDATE) { + + } + } + protected EntityHistory getVersionList(UUID id, Map authHeaders) throws HttpResponseException { WebTarget target = getResource(collectionName + "/" + id + "/versions"); return TestUtils.get(target, EntityHistory.class, authHeaders); } - protected ResultList getChangeEvents(List eventTypes, List entityTypes, Date date, - Map authHeaders) throws HttpResponseException { + protected ResultList getChangeEvents(String entityCreated, String entityUpdated, String entityDeleted, + Date date, Map authHeaders) + throws HttpResponseException { WebTarget target = getResource("events"); - target = target.queryParam("eventTypes", eventTypes); - target = target.queryParam("entityTypes", entityTypes); + target = entityCreated == null ? target : target.queryParam("entityCreated", entityCreated); + target = entityUpdated == null ? target : target.queryParam("entityUpdated", entityUpdated); + target = entityDeleted == null ? target : target.queryParam("entityDeleted", entityDeleted); target = target.queryParam("date", RestUtil.DATE_TIME_FORMAT.format(date)); return TestUtils.get(target, ChangeEventList.class, authHeaders); } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/charts/ChartResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/charts/ChartResourceTest.java index 3dd9cab9151..5502b420441 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/charts/ChartResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/charts/ChartResourceTest.java @@ -65,7 +65,8 @@ public class ChartResourceTest extends EntityResourceTest { public static EntityReference LOOKER_REFERENCE; public ChartResourceTest() { - super(Chart.class, ChartList.class, "charts", ChartResource.FIELDS, true, true, true); + super(Entity.CHART, Chart.class, ChartList.class, "charts", ChartResource.FIELDS, + true, true, true); } @BeforeAll @@ -306,7 +307,7 @@ public class ChartResourceTest extends EntityResourceTest { } @Override - public void validatePatchedEntity(Chart expected, Chart patched, Map authHeaders) { + public void compareEntities(Chart expected, Chart patched, Map authHeaders) { validateCommonEntityFields(getEntityInterface(patched), expected.getDescription(), TestUtils.getPrincipal(authHeaders), expected.getOwner()); assertService(expected.getService(), patched.getService()); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java index e1f9d991c98..06c92a71dd5 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java @@ -78,7 +78,8 @@ public class DashboardResourceTest extends EntityResourceTest { public static List CHART_REFERENCES; public DashboardResourceTest() { - super(Dashboard.class, DashboardList.class, "dashboards", DashboardResource.FIELDS, true, true, true); + super(Entity.DASHBOARD, Dashboard.class, DashboardList.class, "dashboards", DashboardResource.FIELDS, true, + true, true); } @@ -397,7 +398,7 @@ public class DashboardResourceTest extends EntityResourceTest { } @Override - public void validatePatchedEntity(Dashboard expected, Dashboard updated, Map authHeaders) { + public void compareEntities(Dashboard expected, Dashboard updated, Map authHeaders) { } @Override diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/DatabaseResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/DatabaseResourceTest.java index c4ef4fd71c3..5c9f3b534d9 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/DatabaseResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/DatabaseResourceTest.java @@ -54,7 +54,8 @@ import static org.openmetadata.catalog.util.TestUtils.authHeaders; public class DatabaseResourceTest extends EntityResourceTest { public DatabaseResourceTest() { - super(Database.class, DatabaseList.class, "databases", DatabaseResource.FIELDS, false, true, false); + super(Entity.DATABASE, Database.class, DatabaseList.class, "databases", + DatabaseResource.FIELDS, false, true, false); } @BeforeAll @@ -317,7 +318,7 @@ public class DatabaseResourceTest extends EntityResourceTest { } @Override - public void validatePatchedEntity(Database expected, Database updated, Map authHeaders) { + public void compareEntities(Database expected, Database updated, Map authHeaders) { validateCommonEntityFields(getEntityInterface(updated), expected.getDescription(), TestUtils.getPrincipal(authHeaders), expected.getOwner()); // Validate service diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java index ce433e2d015..bae1a31d4a2 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java @@ -130,7 +130,8 @@ public class TableResourceTest extends EntityResourceTest { public TableResourceTest() { - super(Table.class, TableList.class, "tables", TableResource.FIELDS, true, true, true); + super(Entity.TABLE, Table.class, TableList.class, "tables", TableResource.FIELDS, + true, true, true); } @BeforeAll @@ -278,10 +279,12 @@ public class TableResourceTest extends EntityResourceTest
{ CreateTable create1 = create(test, 1).withColumns(Arrays.asList(c1, c2)); Table table1 = createAndCheckEntity(create1, adminAuthHeaders()); - // Test PUT operation + // Test PUT operation - put operation to create CreateTable create2 = create(test, 2).withColumns(Arrays.asList(c1, c2)).withName("put_complexColumnType"); - Table table2= updateAndCheckEntity(create2, CREATED, adminAuthHeaders(), UpdateType.CREATED, null); - // Update without any change + System.out.println("Put to create"); + Table table2 = updateAndCheckEntity(create2, CREATED, adminAuthHeaders(), UpdateType.CREATED, null); + // Test PUT operation again without any change + System.out.println("Put with no change"); ChangeDescription change = getChangeDescription(table2.getVersion()); updateAndCheckEntity(create2, Status.OK, adminAuthHeaders(), NO_CHANGE, change); @@ -1315,7 +1318,7 @@ public class TableResourceTest extends EntityResourceTest
{ } @Override - public void validatePatchedEntity(Table expected, Table patched, Map authHeaders) throws HttpResponseException { + public void compareEntities(Table expected, Table patched, Map authHeaders) throws HttpResponseException { validateCommonEntityFields(getEntityInterface(patched), expected.getDescription(), TestUtils.getPrincipal(authHeaders), expected.getOwner()); @@ -1344,14 +1347,13 @@ public class TableResourceTest extends EntityResourceTest
{ return; } if (fieldName.startsWith("columns") && fieldName.endsWith("constraint")) { - // Column constraint ColumnConstraint expectedConstraint = (ColumnConstraint) expected; ColumnConstraint actualConstraint = ColumnConstraint.fromValue((String) actual); assertEquals(expectedConstraint, actualConstraint); } else if (fieldName.endsWith("tableConstraints")) { - List expectedConstraints = (List) expected; - List actualConstraints = JsonUtils.readObjects(actual.toString(), TableConstraint.class); - assertEquals(expectedConstraints, actualConstraints); + List expectedConstraints = (List) expected; + List actualConstraints = JsonUtils.readObjects(actual.toString(), TableConstraint.class); + assertEquals(expectedConstraints, actualConstraints); } else if (fieldName.contains("columns") && !fieldName.endsWith("tags") && !fieldName.endsWith("description")) { List expectedRefs = (List) expected; List actualRefs = JsonUtils.readObjects(actual.toString(), Column.class); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/pipelines/PipelineResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/pipelines/PipelineResourceTest.java index 9896bb4a722..24d585f861b 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/pipelines/PipelineResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/pipelines/PipelineResourceTest.java @@ -68,7 +68,8 @@ public class PipelineResourceTest extends EntityResourceTest { public static List TASKS; public PipelineResourceTest() { - super(Pipeline.class, PipelineList.class, "pipelines", PipelineResource.FIELDS, true, true, true); + super(Entity.PIPELINE, Pipeline.class, PipelineList.class, "pipelines", PipelineResource.FIELDS, + true, true, true); } @@ -106,7 +107,7 @@ public class PipelineResourceTest extends EntityResourceTest { } @Override - public void validatePatchedEntity(Pipeline expected, Pipeline updated, Map authHeaders) throws HttpResponseException { + public void compareEntities(Pipeline expected, Pipeline updated, Map authHeaders) throws HttpResponseException { validateCommonEntityFields(getEntityInterface(updated), expected.getDescription(), TestUtils.getPrincipal(authHeaders), expected.getOwner()); assertEquals(expected.getDisplayName(), updated.getDisplayName()); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java index cba403930b8..7e0899cec73 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java @@ -71,7 +71,8 @@ public class TeamResourceTest extends EntityResourceTest { final Profile PROFILE = new Profile().withImages(new ImageList().withImage(URI.create("http://image.com"))); public TeamResourceTest() { - super(Team.class, TeamList.class, "teams", TeamResource.FIELDS, false, false, false); + super(Entity.TEAM, Team.class, TeamList.class, "teams", TeamResource.FIELDS, + false, false, false); } @Test @@ -451,7 +452,7 @@ public class TeamResourceTest extends EntityResourceTest { } @Override - public void validatePatchedEntity(Team expected, Team updated, Map authHeaders) { + public void compareEntities(Team expected, Team updated, Map authHeaders) { validateCommonEntityFields(getEntityInterface(updated), expected.getDescription(), TestUtils.getPrincipal(authHeaders), null); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/UserResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/UserResourceTest.java index 2faae6c6d1d..d45ab1c5667 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/UserResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/UserResourceTest.java @@ -83,7 +83,8 @@ public class UserResourceTest extends EntityResourceTest { final Profile PROFILE = new Profile().withImages(new ImageList().withImage(URI.create("http://image.com"))); public UserResourceTest() { - super(User.class, UserList.class, "users", UserResource.FIELDS, false, false, false); + super(Entity.USER, User.class, UserList.class, "users", UserResource.FIELDS, + false, false, false); } @Test @@ -321,7 +322,7 @@ public class UserResourceTest extends EntityResourceTest { assertNull(user.getDisplayName()); assertNull(user.getIsBot()); assertNull(user.getProfile()); - assertNull(user.getDeactivated()); +// assertNull(user.getDeactivated()); assertNull(user.getTimezone()); EntityReference team1 = new TeamEntityInterface(createTeam(TeamResourceTest.create(test, 1), @@ -562,7 +563,7 @@ public class UserResourceTest extends EntityResourceTest { } @Override - public void validatePatchedEntity(User expected, User updated, Map authHeaders) { + public void compareEntities(User expected, User updated, Map authHeaders) { validateCommonEntityFields(getEntityInterface(expected), expected.getDescription(), TestUtils.getPrincipal(authHeaders), null); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/topics/TopicResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/topics/TopicResourceTest.java index 9bbad484c30..f2a0360c0fb 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/topics/TopicResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/topics/TopicResourceTest.java @@ -54,7 +54,8 @@ import static org.openmetadata.catalog.util.TestUtils.authHeaders; public class TopicResourceTest extends EntityResourceTest { public TopicResourceTest() { - super(Topic.class, TopicList.class, "topics", TopicResource.FIELDS, true, true, true); + super(Entity.TOPIC, Topic.class, TopicList.class, "topics", TopicResource.FIELDS, + true, true, true); } @Test @@ -304,7 +305,7 @@ public class TopicResourceTest extends EntityResourceTest { } @Override - public void validatePatchedEntity(Topic expected, Topic updated, Map authHeaders) throws HttpResponseException { + public void compareEntities(Topic expected, Topic updated, Map authHeaders) throws HttpResponseException { validateCommonEntityFields(getEntityInterface(updated), expected.getDescription(), TestUtils.getPrincipal(authHeaders), expected.getOwner()); assertService(expected.getService(), expected.getService());