diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v006__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v006__create_db_connection_info.sql new file mode 100644 index 00000000000..03a6cea37d5 --- /dev/null +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v006__create_db_connection_info.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS web_analytic_event ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL, + eventType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.eventType') NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), + UNIQUE(name), + INDEX name_index (name) +); \ No newline at end of file diff --git a/bootstrap/sql/org.postgresql.Driver/v006__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v006__create_db_connection_info.sql new file mode 100644 index 00000000000..e8702a2ed9e --- /dev/null +++ b/bootstrap/sql/org.postgresql.Driver/v006__create_db_connection_info.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS web_analytic_event ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, + eventType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'eventType') STORED NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, + UNIQUE (name) +); + +CREATE INDEX IF NOT EXISTS name_index ON web_analytic_event(name); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index fb589957471..e0b0550ab8a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -99,6 +99,7 @@ public final class Entity { public static final String TEST_DEFINITION = "testDefinition"; public static final String TEST_SUITE = "testSuite"; public static final String TEST_CASE = "testCase"; + public static final String WEB_ANALYTIC_EVENT = "webAnalyticEvent"; // // Policy entity diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 762370f27e1..8324be869ee 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -44,6 +44,7 @@ import org.jdbi.v3.sqlobject.statement.SqlQuery; import org.jdbi.v3.sqlobject.statement.SqlUpdate; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.TokenInterface; +import org.openmetadata.schema.analytics.WebAnalyticEvent; import org.openmetadata.schema.auth.EmailVerificationToken; import org.openmetadata.schema.auth.PasswordResetToken; import org.openmetadata.schema.auth.RefreshToken; @@ -222,6 +223,9 @@ public interface CollectionDAO { @CreateSqlObject TestCaseDAO testCaseDAO(); + @CreateSqlObject + WebAnalyticEventDAO webAnalyticEventDAO(); + @CreateSqlObject UtilDAO utilDAO(); @@ -2938,6 +2942,23 @@ public interface CollectionDAO { } } + interface WebAnalyticEventDAO extends EntityDAO { + @Override + default String getTableName() { + return "web_analytic_event"; + } + + @Override + default Class getEntityClass() { + return WebAnalyticEvent.class; + } + + @Override + default String getNameColumn() { + return "fullyQualifiedName"; + } + } + interface EntityExtensionTimeSeriesDAO { @ConnectionAwareSqlUpdate( value = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WebAnalyticEventRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WebAnalyticEventRepository.java new file mode 100644 index 00000000000..74b1dff6ec8 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WebAnalyticEventRepository.java @@ -0,0 +1,105 @@ +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.Entity.WEB_ANALYTIC_EVENT; + +import java.io.IOException; +import java.util.List; +import java.util.UUID; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.analytics.WebAnalyticEvent; +import org.openmetadata.schema.analytics.WebAnalyticEventData; +import org.openmetadata.schema.type.ChangeDescription; +import org.openmetadata.schema.type.ChangeEvent; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.EventType; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.ResultList; + +public class WebAnalyticEventRepository extends EntityRepository { + public static final String COLLECTION_PATH = "/v1/analytics/webAnalyticEvent"; + private static final String UPDATE_FIELDS = "owner"; + private static final String PATCH_FIELDS = "owner"; + private static final String WEB_ANALYTICS_EVENT_DATA_EXTENSION = "webAnalyticEvent.webAnalyticEventData"; + + public WebAnalyticEventRepository(CollectionDAO dao) { + super( + COLLECTION_PATH, + WEB_ANALYTIC_EVENT, + WebAnalyticEvent.class, + dao.webAnalyticEventDAO(), + dao, + PATCH_FIELDS, + UPDATE_FIELDS); + } + + @Override + public WebAnalyticEvent setFields(WebAnalyticEvent entity, EntityUtil.Fields fields) throws IOException { + entity.setOwner(fields.contains("owner") ? getOwner(entity) : null); + return entity; + } + + @Override + public void prepare(WebAnalyticEvent entity) throws IOException { + entity.setFullyQualifiedName(entity.getName()); + } + + @Override + public void storeEntity(WebAnalyticEvent entity, boolean update) throws IOException { + EntityReference owner = entity.getOwner(); + + entity.withOwner(null).withHref(null); + store(entity.getId(), entity, update); + + entity.withOwner(owner); + } + + @Override + public void storeRelationships(WebAnalyticEvent entity) throws IOException { + storeOwner(entity, entity.getOwner()); + } + + private ChangeEvent getChangeEvent( + EntityInterface updated, ChangeDescription change, String entityType, Double prevVersion) { + return new ChangeEvent() + .withEntity(updated) + .withChangeDescription(change) + .withEventType(EventType.ENTITY_UPDATED) + .withEntityType(entityType) + .withTimestamp(System.currentTimeMillis()) + .withPreviousVersion(prevVersion); + } + + @Transaction + public Response addWebAnalyticEventData(UriInfo uriInfo, WebAnalyticEventData webAnalyticEventData) + throws IOException { + webAnalyticEventData.setEventId(UUID.randomUUID()); + WebAnalyticEvent webAnalyticEvent = dao.findEntityByName(webAnalyticEventData.getEventType().value()); + daoCollection + .entityExtensionTimeSeriesDao() + .insert( + webAnalyticEventData.getEventType().value(), + WEB_ANALYTICS_EVENT_DATA_EXTENSION, + "webAnalyticEventData", + JsonUtils.pojoToJson(webAnalyticEventData)); + + return Response.ok(webAnalyticEventData).build(); + } + + public ResultList getWebAnalyticEventData(String eventType, Long startTs, Long endTs) + throws IOException { + List webAnalyticEventData; + webAnalyticEventData = + JsonUtils.readObjects( + daoCollection + .entityExtensionTimeSeriesDao() + .listBetweenTimestamps(eventType, WEB_ANALYTICS_EVENT_DATA_EXTENSION, startTs, endTs), + WebAnalyticEventData.class); + + return new ResultList<>( + webAnalyticEventData, String.valueOf(startTs), String.valueOf(endTs), webAnalyticEventData.size()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java new file mode 100644 index 00000000000..402dbfef8bd --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java @@ -0,0 +1,445 @@ +package org.openmetadata.service.resources.analytics; + +import com.google.inject.Inject; +import io.swagger.annotations.Api; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import java.io.IOException; +import java.util.List; +import java.util.UUID; +import javax.json.JsonPatch; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.PATCH; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.analytics.WebAnalyticEvent; +import org.openmetadata.schema.analytics.WebAnalyticEventData; +import org.openmetadata.schema.api.tests.CreateWebAnalyticEvent; +import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.jdbi3.WebAnalyticEventRepository; +import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.resources.EntityResource; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.util.RestUtil; +import org.openmetadata.service.util.ResultList; + +@Slf4j +@Path("/v1/analytics/webAnalyticEvent") +@Api(value = "webAnalyticEvent collection", tags = "webAnalyticEvent collection") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "WebAnalyticEvent") +public class WebAnalyticEventResource extends EntityResource { + public static final String COLLECTION_PATH = WebAnalyticEventRepository.COLLECTION_PATH; + static final String FIELDS = "owner"; + private final WebAnalyticEventRepository daoWebAnalyticEvent; + + @Override + public WebAnalyticEvent addHref(UriInfo uriInfo, WebAnalyticEvent entity) { + entity.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, entity.getId())); + Entity.withHref(uriInfo, entity.getOwner()); + return entity; + } + + @Inject + public WebAnalyticEventResource(CollectionDAO dao, Authorizer authorizer) { + super(WebAnalyticEvent.class, new WebAnalyticEventRepository(dao), authorizer); + this.daoWebAnalyticEvent = new WebAnalyticEventRepository(dao); + } + + public static class WebAnalyticEventList extends ResultList { + @SuppressWarnings("unused") + public WebAnalyticEventList() { + // Empty constructor needed for deserialization + } + + public WebAnalyticEventList(List data, String beforeCursor, String afterCursor, int total) { + super(data, beforeCursor, afterCursor, total); + } + } + + public static class WebAnalyticEventDataList extends ResultList { + @SuppressWarnings("unused") + public WebAnalyticEventDataList() { + // Empty constructor needed for deserialization + } + + public WebAnalyticEventDataList( + List data, String beforeCursor, String afterCursor, int total) { + super(data, beforeCursor, afterCursor, total); + } + } + + @SuppressWarnings("unused") // Method used for reflection of webAnalyticEventTypes + public void initialize(OpenMetadataApplicationConfig config) throws IOException { + // Find the existing webAnalyticEventTypes and add them from json files + List webAnalyticEvents = + dao.getEntitiesFromSeedData(".*json/data/analytics/webAnalyticEvents/.*\\.json$"); + for (WebAnalyticEvent webAnalyticEvent : webAnalyticEvents) { + dao.initializeEntity(webAnalyticEvent); + } + } + + @GET + @Operation( + operationId = "listWebAnalyticEventTypes", + summary = "List web analytic event types", + tags = "WebAnalyticEvent", + description = + "Get a list of web analytics event types." + + "Use field parameter to get only necessary fields. Use cursor-based pagination to limit the number " + + "entries in the list using `limit` and `before` or `after` query params.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of web analytic event types", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = WebAnalyticEventResource.WebAnalyticEventList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter(description = "Limit the number report Definition returned. (1 to 1000000, default = " + "10)") + @DefaultValue("10") + @QueryParam("limit") + @Min(0) + @Max(1000000) + int limitParam, + @Parameter( + description = "Returns list of report definitions before this cursor", + schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter( + description = "Returns list of report definitions after this cursor", + schema = @Schema(type = "string")) + @QueryParam("after") + String after, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException { + ListFilter filter = new ListFilter(include); + return super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after); + } + + @POST + @Operation( + operationId = "createWebAnalyticEventType", + summary = "Create a web analytic event type", + tags = "WebAnalyticEvent", + description = "Create a web analytic event type", + responses = { + @ApiResponse( + responseCode = "200", + description = "Create a web analytic event type", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = WebAnalyticEvent.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response create( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateWebAnalyticEvent create) + throws IOException { + WebAnalyticEvent webAnalyticEvent = getWebAnalyticEvent(create, securityContext.getUserPrincipal().getName()); + return create(uriInfo, securityContext, webAnalyticEvent, true); + } + + @PUT + @Operation( + operationId = "createOrUpdateWebAnalyticEventType", + summary = "Update a web analytic event type", + tags = "WebAnalyticEvent", + description = "Update web analytic event type.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Updated web analytic event type", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = WebAnalyticEvent.class))) + }) + public Response createOrUpdate( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateWebAnalyticEvent create) + throws IOException { + WebAnalyticEvent webAnalyticEvent = getWebAnalyticEvent(create, securityContext.getUserPrincipal().getName()); + return createOrUpdate(uriInfo, securityContext, webAnalyticEvent, true); + } + + @GET + @Path("/{id}") + @Operation( + operationId = "getWebAnalyticEventTypeById", + summary = "Get a web analytic event type by id", + tags = "WebAnalyticEvent", + description = "Get a web analytic event type by `ID`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "A web analytic event type", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = WebAnalyticEvent.class))), + @ApiResponse(responseCode = "404", description = "Web Analytic Event for instance {id} is not found") + }) + public WebAnalyticEvent get( + @Context UriInfo uriInfo, + @PathParam("id") UUID id, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException { + return getInternal(uriInfo, securityContext, id, fieldsParam, include); + } + + @PATCH + @Path("/{id}") + @Operation( + operationId = "patchWebAnalyticEventTypeById", + summary = "Update a web analytic event type", + tags = "WebAnalyticEvent", + description = "Update a web analytic event type.", + externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902")) + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + public Response updateDescription( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") UUID id, + @RequestBody( + description = "JsonPatch with array of operations", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON_PATCH_JSON, + examples = { + @ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]") + })) + JsonPatch patch) + throws IOException { + return patchInternal(uriInfo, securityContext, id, patch); + } + + @DELETE + @Path("/{id}") + @Operation( + operationId = "deleteWebAnalyticEventTypeById", + summary = "delete a web analytic event type", + tags = "WebAnalyticEvent", + description = "Delete a web analytic event type by id.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "Web Analytic event for instance {id} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Hard delete the entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "Web Analytic event Id", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) + throws IOException { + return delete(uriInfo, securityContext, id, false, hardDelete, true); + } + + @GET + @Path("/name/{name}") + @Operation( + operationId = "getWebAnalyticEventTypeByName", + summary = "Get a web analytic event type by Name", + tags = "WebAnalyticEvent", + description = "Get a web analytic event type by Name.", + responses = { + @ApiResponse( + responseCode = "200", + description = "A web analytic event type", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = WebAnalyticEvent.class))), + @ApiResponse(responseCode = "404", description = "Web Analytic event type for instance {id} is not found") + }) + public WebAnalyticEvent getByName( + @Context UriInfo uriInfo, + @PathParam("name") String name, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException { + return getByNameInternal(uriInfo, securityContext, name, fieldsParam, include); + } + + @GET + @Path("/{id}/versions") + @Operation( + operationId = "listAllWebAnalyticEventTypeVersion", + summary = "List web analytic event type versions", + tags = "WebAnalyticEvent", + description = "Get a list of all the version of a web analytic event type by `id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List all web analytic event type versions", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class))) + }) + public EntityHistory listVersions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Web Analytic event type Id", schema = @Schema(type = "string")) @PathParam("id") + UUID id) + throws IOException { + return super.listVersionsInternal(securityContext, id); + } + + @GET + @Path("/{id}/versions/{version}") + @Operation( + operationId = "getSpecificWebAnalyticEventTypeVersion", + summary = "Get a version of the report definition", + tags = "WebAnalyticEvent", + description = "Get a version of the web analytic event type by `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "WebAnalyticEvent", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = WebAnalyticEvent.class))), + @ApiResponse( + responseCode = "404", + description = "Web Analytic event type for instance {id} and version {version} is " + "not found") + }) + public WebAnalyticEvent getVersion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Web Analytic Event type Id", schema = @Schema(type = "string")) @PathParam("id") + UUID id, + @Parameter( + description = "Web Analytic Event type version number in the form `major`.`minor`", + schema = @Schema(type = "string", example = "0.1 or 1.1")) + @PathParam("version") + String version) + throws IOException { + return super.getVersionInternal(securityContext, id, version); + } + + @PUT + @Path("/collect") + @Operation( + operationId = "addWebAnalyticEventData", + summary = "Add web analytic event data", + tags = "WebAnalyticEvent", + description = "Add web analytic event data", + responses = { + @ApiResponse( + responseCode = "200", + description = "Successfully added web analytic event data", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = WebAnalyticEventData.class))) + }) + public Response addReportResult( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid WebAnalyticEventData webAnalyticEventData) + throws IOException { + authorizer.authorizeAdmin(securityContext, true); + return dao.addWebAnalyticEventData(uriInfo, webAnalyticEventData); + } + + @GET + @Path("/collect") + @Operation( + operationId = "getWebAnalyticEventData", + summary = "Retrieve web analytic data", + tags = "WebAnalyticEvent", + description = "Retrieve web analytic data.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of web analytic data", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = WebAnalyticEventResource.WebAnalyticEventDataList.class))) + }) + public ResultList listWebAnalyticEventData( + @Context SecurityContext securityContext, + @Parameter( + description = "Filter web analytic events for a particular event type", + schema = @Schema(type = "string")) + @NonNull + @QueryParam("eventType") + String eventType, + @Parameter( + description = "Filter web analytic events after the given start timestamp", + schema = @Schema(type = "number")) + @NonNull + @QueryParam("startTs") + Long startTs, + @Parameter( + description = "Filter web analytic events before the given end timestamp", + schema = @Schema(type = "number")) + @NonNull + @QueryParam("endTs") + Long endTs) + throws IOException { + return dao.getWebAnalyticEventData(eventType, startTs, endTs); + } + + private WebAnalyticEvent getWebAnalyticEvent(CreateWebAnalyticEvent create, String user) throws IOException { + return copy(new WebAnalyticEvent(), create, user) + .withName(create.getName()) + .withDisplayName(create.getDisplayName()) + .withDescription(create.getDescription()) + .withEventType(create.getEventType()); + } +} diff --git a/openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json b/openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json index 5e6bab05cdb..4b50b4c797f 100644 --- a/openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json +++ b/openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json @@ -438,5 +438,16 @@ "Delete", "ViewAll" ] + }, + { + "name" : "webAnalyticEvent", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditDisplayName" + ] } ] \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/analytics/webAnalyticEvents/pageView.json b/openmetadata-service/src/main/resources/json/data/analytics/webAnalyticEvents/pageView.json new file mode 100644 index 00000000000..dd46b5c1b7d --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/analytics/webAnalyticEvents/pageView.json @@ -0,0 +1,5 @@ +{ + "name": "PageView", + "description": "Captures web analytic page view events", + "eventType": "PageView" +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResourceTest.java new file mode 100644 index 00000000000..d0deac78420 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResourceTest.java @@ -0,0 +1,145 @@ +package org.openmetadata.service.resources.analytics; + +import static javax.ws.rs.core.Response.Status.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; +import static org.openmetadata.service.util.TestUtils.assertResponseContains; + +import java.io.IOException; +import java.text.ParseException; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import javax.ws.rs.client.WebTarget; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openmetadata.schema.analytics.PageViewData; +import org.openmetadata.schema.analytics.WebAnalyticEvent; +import org.openmetadata.schema.analytics.WebAnalyticEventData; +import org.openmetadata.schema.analytics.type.WebAnalyticEventType; +import org.openmetadata.schema.api.tests.CreateWebAnalyticEvent; +import org.openmetadata.service.Entity; +import org.openmetadata.service.OpenMetadataApplicationTest; +import org.openmetadata.service.resources.EntityResourceTest; +import org.openmetadata.service.util.ResultList; +import org.openmetadata.service.util.TestUtils; + +public class WebAnalyticEventResourceTest extends EntityResourceTest { + public WebAnalyticEventResourceTest() { + super( + Entity.WEB_ANALYTIC_EVENT, + WebAnalyticEvent.class, + WebAnalyticEventResource.WebAnalyticEventList.class, + "analytics/webAnalyticEvent", + WebAnalyticEventResource.FIELDS); + supportsEmptyDescription = false; + supportsFollowers = false; + supportsAuthorizedMetadataOperations = false; + supportsOwner = false; + } + + @Test + void post_web_analytic_event_200(TestInfo test) throws IOException { + CreateWebAnalyticEvent create = createRequest(test); + create.withName("bar").withEventType(WebAnalyticEventType.PAGE_VIEW); + WebAnalyticEvent webAnalyticEvent = createAndCheckEntity(create, ADMIN_AUTH_HEADERS); + webAnalyticEvent = getEntity(webAnalyticEvent.getId(), ADMIN_AUTH_HEADERS); + validateCreatedEntity(webAnalyticEvent, create, ADMIN_AUTH_HEADERS); + } + + @Test + void post_web_analytic_event_4x(TestInfo test) throws IOException { + assertResponseContains( + () -> createEntity(createRequest(test).withEventType(null), ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "eventType must not be null"); + + assertResponseContains( + () -> createEntity(createRequest(test).withName(null), ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "name must not be null"); + } + + @Test + void put_web_analytic_event_data_200(TestInfo test) throws IOException, ParseException { + WebAnalyticEventData webAnalyticEventData = + new WebAnalyticEventData() + .withTimestamp(TestUtils.dateToTimestamp("2022-10-11")) + .withEventType(WebAnalyticEventType.PAGE_VIEW) + .withEventData( + new PageViewData() + .withHostname("http://localhost") + .withUserId(UUID.randomUUID()) + .withSessionId(UUID.randomUUID())); + + putWebAnalyticEventData(webAnalyticEventData, ADMIN_AUTH_HEADERS); + + ResultList webAnalyticEventDataResultList = + getWebAnalyticEventData( + WebAnalyticEventType.PAGE_VIEW.value(), + TestUtils.dateToTimestamp("2022-10-10"), + TestUtils.dateToTimestamp("2022-10-12"), + ADMIN_AUTH_HEADERS); + + verifyWebAnalyticEventData(webAnalyticEventDataResultList, List.of(webAnalyticEventData), 1); + } + + @Override + public CreateWebAnalyticEvent createRequest(String name) { + return new CreateWebAnalyticEvent() + .withName(name) + .withDescription(name) + .withEventType(WebAnalyticEventType.PAGE_VIEW); + } + + @Override + public void validateCreatedEntity( + WebAnalyticEvent createdEntity, CreateWebAnalyticEvent request, Map authHeaders) + throws HttpResponseException { + assertEquals(request.getName(), createdEntity.getName()); + assertEquals(request.getDescription(), createdEntity.getDescription()); + } + + @Override + public void compareEntities(WebAnalyticEvent expected, WebAnalyticEvent updated, Map authHeaders) + throws HttpResponseException { + assertEquals(expected.getName(), updated.getName()); + assertEquals(expected.getFullyQualifiedName(), updated.getFullyQualifiedName()); + assertEquals(expected.getDescription(), updated.getDescription()); + } + + @Override + public WebAnalyticEvent validateGetWithDifferentFields(WebAnalyticEvent entity, boolean byName) + throws HttpResponseException { + return null; + } + + @Override + public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException { + return; + } + + public static void putWebAnalyticEventData(WebAnalyticEventData data, Map authHeaders) + throws HttpResponseException { + WebTarget target = OpenMetadataApplicationTest.getResource("analytics/webAnalyticEvent/collect"); + TestUtils.put(target, data, OK, authHeaders); + } + + public static ResultList getWebAnalyticEventData( + String eventType, Long start, Long end, Map authHeaders) throws HttpResponseException { + WebTarget target = OpenMetadataApplicationTest.getResource("analytics/webAnalyticEvent/collect"); + target = target.queryParam("startTs", start); + target = target.queryParam("endTs", end); + target = target.queryParam("eventType", eventType); + return TestUtils.get(target, WebAnalyticEventResource.WebAnalyticEventDataList.class, authHeaders); + } + + private void verifyWebAnalyticEventData( + ResultList actualWebAnalyticEventData, + List expectedWebAnalyticEventData, + int expectedCount) { + assertEquals(expectedCount, actualWebAnalyticEventData.getPaging().getTotal()); + assertEquals(expectedWebAnalyticEventData.size(), actualWebAnalyticEventData.getData().size()); + } +} diff --git a/openmetadata-spec/src/main/resources/json/schema/analytics/basic.json b/openmetadata-spec/src/main/resources/json/schema/analytics/basic.json new file mode 100644 index 00000000000..7d971e45538 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/analytics/basic.json @@ -0,0 +1,16 @@ +{ + "$id": "https://open-metadata.org/schema/analytics/basic.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Basic", + "description": "This schema defines basic types that are used by analytics classes", + "definitions": { + "webAnalyticEventType": { + "javaType": "org.openmetadata.schema.analytics.type.WebAnalyticEventType", + "description": "event type", + "type": "string", + "enum": [ + "PageView" + ] + } + } + } \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/analytics/pageViewEvent.json b/openmetadata-spec/src/main/resources/json/schema/analytics/pageViewEvent.json new file mode 100644 index 00000000000..2b2affe8879 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/analytics/pageViewEvent.json @@ -0,0 +1,46 @@ +{ + "$id": "https://open-metadata.org/schema/analytics/pageViewData.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "pageViewData", + "type": "object", + "javaType": "org.openmetadata.schema.analytics.PageViewData", + "description": "Page view data event", + "properties": { + "fullUrl": { + "description": "complete URL of the page", + "type": "string" + }, + "url": { + "description": "url part after the domain specification", + "type": "string" + }, + "hostname": { + "description": "domain name", + "type": "string" + }, + "language": { + "description": "language set on the page", + "type": "string" + }, + "screenSize": { + "description": "Size of the screen", + "type": "string" + }, + "userId": { + "description": "OpenMetadata logged in user Id", + "$ref": "../type/basic.json#/definitions/uuid" + }, + "sessionId": { + "description": "Unique ID identifying a session", + "$ref": "../type/basic.json#/definitions/uuid" + }, + "pageLoadTime": { + "description": "time for the page to load in seconds", + "type": "number" + }, + "referrer": { + "description": "referrer URL", + "type": "string" + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEvent.json b/openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEvent.json new file mode 100644 index 00000000000..b69f6e97919 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEvent.json @@ -0,0 +1,72 @@ +{ + "$id": "https://open-metadata.org/schema/analytics/webAnalyticEvent.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "WebAnalyticEvent", + "description": "Web Analytic Event", + "type": "object", + "javaType": "org.openmetadata.schema.analytics.WebAnalyticEvent", + "javaInterfaces": ["org.openmetadata.schema.EntityInterface"], + "properties": { + "id": { + "description": "Unique identifier of the report.", + "$ref": "../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name that identifies this event.", + "$ref": "../type/basic.json#/definitions/entityName" + }, + "fullyQualifiedName": { + "description": "FullyQualifiedName same as `name`.", + "$ref": "../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "displayName": { + "description": "Display Name that identifies this web analytics event.", + "type": "string" + }, + "description": { + "description": "Description of the event.", + "$ref": "../type/basic.json#/definitions/markdown" + }, + "eventType": { + "description": "event type", + "$ref": "./basic.json#/definitions/webAnalyticEventType" + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../type/entityHistory.json#/definitions/entityVersion" + }, + "owner": { + "description": "Owner of this report.", + "$ref": "../type/entityReference.json", + "default": null + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who performed the update.", + "type": "string" + }, + "href": { + "description": "Link to the resource corresponding to this entity.", + "$ref": "../type/basic.json#/definitions/href" + }, + "changeDescription": { + "description": "Change that lead to this version of the entity.", + "$ref": "../type/entityHistory.json#/definitions/changeDescription" + }, + "deleted": { + "description": "When `true` indicates the entity has been soft deleted.", + "type": "boolean", + "default": false + }, + "enabled": { + "description": "Weather the event is enable (i.e. data is being collected)", + "type": "boolean", + "default": true + } + }, + "required": ["eventType", "name"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEventData.json b/openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEventData.json new file mode 100644 index 00000000000..ae8467bc295 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEventData.json @@ -0,0 +1,30 @@ +{ + "$id": "https://open-metadata.org/schema/analytics/webAnalyticEventData.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "webAnalyticEventData", + "description": "web analytics event data", + "type": "object", + "javaType": "org.openmetadata.schema.analytics.WebAnalyticEventData", + "properties": { + "eventId": { + "description": "Unique identifier of the report.", + "$ref": "../type/basic.json#/definitions/uuid" + }, + "timestamp": { + "description": "event timestamp", + "$ref": "../type/basic.json#/definitions/timestamp" + }, + "eventType": { + "description": "event type", + "$ref": "./basic.json#/definitions/webAnalyticEventType" + }, + "eventData": { + "description": "Web analytic data captured", + "oneOf": [ + {"$ref": "./pageViewEvent.json"} + ] + } + }, + "required": ["eventType"], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/api/analytics/createWebAnalyticEvent.json b/openmetadata-spec/src/main/resources/json/schema/api/analytics/createWebAnalyticEvent.json new file mode 100644 index 00000000000..d03988b2713 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/analytics/createWebAnalyticEvent.json @@ -0,0 +1,33 @@ +{ + "$id": "https://open-metadata.org/schema/api/tests/createWebAnalyticEvent.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateWebAnalyticEvent", + "description": "Payload to create a web analytic event", + "type": "object", + "javaType": "org.openmetadata.schema.api.tests.CreateWebAnalyticEvent", + "javaInterfaces": ["org.openmetadata.schema.CreateEntity"], + "properties": { + "name": { + "description": "Name that identifies this report definition.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display Name the report definition.", + "type": "string" + }, + "description": { + "description": "Description of the report definition.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "eventType": { + "description": "dimension(s) and metric(s) for a report", + "$ref": "../../analytics/basic.json#/definitions/webAnalyticEventType" + }, + "owner": { + "description": "Owner of this report definition", + "$ref": "../../type/entityReference.json" + } + }, + "required": ["name", "eventType"], + "additionalProperties": false +}