Fixes #8154 - Add collect endpoint for web analytic events (#8193)

* Initial implementation of report endpoint

* Modified model files

* Added migration

* Added tests for analytics/report endpoint

* Finalized implementation of the reports endpoint

* Fixed maven test failure

* Fixed java style

* Added policy for reportDefinition

* Fixed minor typos in code

* Added database migration for web analytic event type

* Added logic for collect endpoint

* Added JSON Schema for collect endpoint

* Added index on table columns + added resource description for web event entities

* Fixed policy tests

* Updated add web analytic function to return the body of the request + updated test

* Added referer URL to PageView event

* Changed collection name to align with API endpoint

* updated initSeed method to initializeEntity

* cleaned up PR by removing unused filed from branch

* Fixed requested changes from review

* Clean up names

* Fixed comma in SQL statement
This commit is contained in:
Teddy 2022-10-20 16:38:54 +02:00 committed by GitHub
parent 816111a671
commit 342e4a24da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 955 additions and 0 deletions

View File

@ -0,0 +1,12 @@
CREATE TABLE IF NOT EXISTS web_analytic_event (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
eventType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.eventType') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
UNIQUE(name),
INDEX name_index (name)
);

View File

@ -0,0 +1,13 @@
CREATE TABLE IF NOT EXISTS web_analytic_event (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
eventType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'eventType') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
UNIQUE (name)
);
CREATE INDEX IF NOT EXISTS name_index ON web_analytic_event(name);

View File

@ -99,6 +99,7 @@ public final class Entity {
public static final String TEST_DEFINITION = "testDefinition"; public static final String TEST_DEFINITION = "testDefinition";
public static final String TEST_SUITE = "testSuite"; public static final String TEST_SUITE = "testSuite";
public static final String TEST_CASE = "testCase"; public static final String TEST_CASE = "testCase";
public static final String WEB_ANALYTIC_EVENT = "webAnalyticEvent";
// //
// Policy entity // Policy entity

View File

@ -44,6 +44,7 @@ import org.jdbi.v3.sqlobject.statement.SqlQuery;
import org.jdbi.v3.sqlobject.statement.SqlUpdate; import org.jdbi.v3.sqlobject.statement.SqlUpdate;
import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.TokenInterface; import org.openmetadata.schema.TokenInterface;
import org.openmetadata.schema.analytics.WebAnalyticEvent;
import org.openmetadata.schema.auth.EmailVerificationToken; import org.openmetadata.schema.auth.EmailVerificationToken;
import org.openmetadata.schema.auth.PasswordResetToken; import org.openmetadata.schema.auth.PasswordResetToken;
import org.openmetadata.schema.auth.RefreshToken; import org.openmetadata.schema.auth.RefreshToken;
@ -222,6 +223,9 @@ public interface CollectionDAO {
@CreateSqlObject @CreateSqlObject
TestCaseDAO testCaseDAO(); TestCaseDAO testCaseDAO();
@CreateSqlObject
WebAnalyticEventDAO webAnalyticEventDAO();
@CreateSqlObject @CreateSqlObject
UtilDAO utilDAO(); UtilDAO utilDAO();
@ -2938,6 +2942,23 @@ public interface CollectionDAO {
} }
} }
interface WebAnalyticEventDAO extends EntityDAO<WebAnalyticEvent> {
@Override
default String getTableName() {
return "web_analytic_event";
}
@Override
default Class<WebAnalyticEvent> getEntityClass() {
return WebAnalyticEvent.class;
}
@Override
default String getNameColumn() {
return "fullyQualifiedName";
}
}
interface EntityExtensionTimeSeriesDAO { interface EntityExtensionTimeSeriesDAO {
@ConnectionAwareSqlUpdate( @ConnectionAwareSqlUpdate(
value = value =

View File

@ -0,0 +1,105 @@
package org.openmetadata.service.jdbi3;
import static org.openmetadata.service.Entity.WEB_ANALYTIC_EVENT;
import java.io.IOException;
import java.util.List;
import java.util.UUID;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.analytics.WebAnalyticEvent;
import org.openmetadata.schema.analytics.WebAnalyticEventData;
import org.openmetadata.schema.type.ChangeDescription;
import org.openmetadata.schema.type.ChangeEvent;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.EventType;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ResultList;
public class WebAnalyticEventRepository extends EntityRepository<WebAnalyticEvent> {
public static final String COLLECTION_PATH = "/v1/analytics/webAnalyticEvent";
private static final String UPDATE_FIELDS = "owner";
private static final String PATCH_FIELDS = "owner";
private static final String WEB_ANALYTICS_EVENT_DATA_EXTENSION = "webAnalyticEvent.webAnalyticEventData";
public WebAnalyticEventRepository(CollectionDAO dao) {
super(
COLLECTION_PATH,
WEB_ANALYTIC_EVENT,
WebAnalyticEvent.class,
dao.webAnalyticEventDAO(),
dao,
PATCH_FIELDS,
UPDATE_FIELDS);
}
@Override
public WebAnalyticEvent setFields(WebAnalyticEvent entity, EntityUtil.Fields fields) throws IOException {
entity.setOwner(fields.contains("owner") ? getOwner(entity) : null);
return entity;
}
@Override
public void prepare(WebAnalyticEvent entity) throws IOException {
entity.setFullyQualifiedName(entity.getName());
}
@Override
public void storeEntity(WebAnalyticEvent entity, boolean update) throws IOException {
EntityReference owner = entity.getOwner();
entity.withOwner(null).withHref(null);
store(entity.getId(), entity, update);
entity.withOwner(owner);
}
@Override
public void storeRelationships(WebAnalyticEvent entity) throws IOException {
storeOwner(entity, entity.getOwner());
}
private ChangeEvent getChangeEvent(
EntityInterface updated, ChangeDescription change, String entityType, Double prevVersion) {
return new ChangeEvent()
.withEntity(updated)
.withChangeDescription(change)
.withEventType(EventType.ENTITY_UPDATED)
.withEntityType(entityType)
.withTimestamp(System.currentTimeMillis())
.withPreviousVersion(prevVersion);
}
@Transaction
public Response addWebAnalyticEventData(UriInfo uriInfo, WebAnalyticEventData webAnalyticEventData)
throws IOException {
webAnalyticEventData.setEventId(UUID.randomUUID());
WebAnalyticEvent webAnalyticEvent = dao.findEntityByName(webAnalyticEventData.getEventType().value());
daoCollection
.entityExtensionTimeSeriesDao()
.insert(
webAnalyticEventData.getEventType().value(),
WEB_ANALYTICS_EVENT_DATA_EXTENSION,
"webAnalyticEventData",
JsonUtils.pojoToJson(webAnalyticEventData));
return Response.ok(webAnalyticEventData).build();
}
public ResultList<WebAnalyticEventData> getWebAnalyticEventData(String eventType, Long startTs, Long endTs)
throws IOException {
List<WebAnalyticEventData> webAnalyticEventData;
webAnalyticEventData =
JsonUtils.readObjects(
daoCollection
.entityExtensionTimeSeriesDao()
.listBetweenTimestamps(eventType, WEB_ANALYTICS_EVENT_DATA_EXTENSION, startTs, endTs),
WebAnalyticEventData.class);
return new ResultList<>(
webAnalyticEventData, String.valueOf(startTs), String.valueOf(endTs), webAnalyticEventData.size());
}
}

View File

@ -0,0 +1,445 @@
package org.openmetadata.service.resources.analytics;
import com.google.inject.Inject;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.io.IOException;
import java.util.List;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.PATCH;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.analytics.WebAnalyticEvent;
import org.openmetadata.schema.analytics.WebAnalyticEventData;
import org.openmetadata.schema.api.tests.CreateWebAnalyticEvent;
import org.openmetadata.schema.type.EntityHistory;
import org.openmetadata.schema.type.Include;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.WebAnalyticEventRepository;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.util.RestUtil;
import org.openmetadata.service.util.ResultList;
@Slf4j
@Path("/v1/analytics/webAnalyticEvent")
@Api(value = "webAnalyticEvent collection", tags = "webAnalyticEvent collection")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "WebAnalyticEvent")
public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, WebAnalyticEventRepository> {
public static final String COLLECTION_PATH = WebAnalyticEventRepository.COLLECTION_PATH;
static final String FIELDS = "owner";
private final WebAnalyticEventRepository daoWebAnalyticEvent;
@Override
public WebAnalyticEvent addHref(UriInfo uriInfo, WebAnalyticEvent entity) {
entity.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, entity.getId()));
Entity.withHref(uriInfo, entity.getOwner());
return entity;
}
@Inject
public WebAnalyticEventResource(CollectionDAO dao, Authorizer authorizer) {
super(WebAnalyticEvent.class, new WebAnalyticEventRepository(dao), authorizer);
this.daoWebAnalyticEvent = new WebAnalyticEventRepository(dao);
}
public static class WebAnalyticEventList extends ResultList<WebAnalyticEvent> {
@SuppressWarnings("unused")
public WebAnalyticEventList() {
// Empty constructor needed for deserialization
}
public WebAnalyticEventList(List<WebAnalyticEvent> data, String beforeCursor, String afterCursor, int total) {
super(data, beforeCursor, afterCursor, total);
}
}
public static class WebAnalyticEventDataList extends ResultList<WebAnalyticEventData> {
@SuppressWarnings("unused")
public WebAnalyticEventDataList() {
// Empty constructor needed for deserialization
}
public WebAnalyticEventDataList(
List<WebAnalyticEventData> data, String beforeCursor, String afterCursor, int total) {
super(data, beforeCursor, afterCursor, total);
}
}
@SuppressWarnings("unused") // Method used for reflection of webAnalyticEventTypes
public void initialize(OpenMetadataApplicationConfig config) throws IOException {
// Find the existing webAnalyticEventTypes and add them from json files
List<WebAnalyticEvent> webAnalyticEvents =
dao.getEntitiesFromSeedData(".*json/data/analytics/webAnalyticEvents/.*\\.json$");
for (WebAnalyticEvent webAnalyticEvent : webAnalyticEvents) {
dao.initializeEntity(webAnalyticEvent);
}
}
@GET
@Operation(
operationId = "listWebAnalyticEventTypes",
summary = "List web analytic event types",
tags = "WebAnalyticEvent",
description =
"Get a list of web analytics event types."
+ "Use field parameter to get only necessary fields. Use cursor-based pagination to limit the number "
+ "entries in the list using `limit` and `before` or `after` query params.",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of web analytic event types",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = WebAnalyticEventResource.WebAnalyticEventList.class)))
})
public ResultList<WebAnalyticEvent> list(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(description = "Limit the number report Definition returned. (1 to 1000000, default = " + "10)")
@DefaultValue("10")
@QueryParam("limit")
@Min(0)
@Max(1000000)
int limitParam,
@Parameter(
description = "Returns list of report definitions before this cursor",
schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(
description = "Returns list of report definitions after this cursor",
schema = @Schema(type = "string"))
@QueryParam("after")
String after,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException {
ListFilter filter = new ListFilter(include);
return super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after);
}
@POST
@Operation(
operationId = "createWebAnalyticEventType",
summary = "Create a web analytic event type",
tags = "WebAnalyticEvent",
description = "Create a web analytic event type",
responses = {
@ApiResponse(
responseCode = "200",
description = "Create a web analytic event type",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = WebAnalyticEvent.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response create(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateWebAnalyticEvent create)
throws IOException {
WebAnalyticEvent webAnalyticEvent = getWebAnalyticEvent(create, securityContext.getUserPrincipal().getName());
return create(uriInfo, securityContext, webAnalyticEvent, true);
}
@PUT
@Operation(
operationId = "createOrUpdateWebAnalyticEventType",
summary = "Update a web analytic event type",
tags = "WebAnalyticEvent",
description = "Update web analytic event type.",
responses = {
@ApiResponse(
responseCode = "200",
description = "Updated web analytic event type",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = WebAnalyticEvent.class)))
})
public Response createOrUpdate(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateWebAnalyticEvent create)
throws IOException {
WebAnalyticEvent webAnalyticEvent = getWebAnalyticEvent(create, securityContext.getUserPrincipal().getName());
return createOrUpdate(uriInfo, securityContext, webAnalyticEvent, true);
}
@GET
@Path("/{id}")
@Operation(
operationId = "getWebAnalyticEventTypeById",
summary = "Get a web analytic event type by id",
tags = "WebAnalyticEvent",
description = "Get a web analytic event type by `ID`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "A web analytic event type",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = WebAnalyticEvent.class))),
@ApiResponse(responseCode = "404", description = "Web Analytic Event for instance {id} is not found")
})
public WebAnalyticEvent get(
@Context UriInfo uriInfo,
@PathParam("id") UUID id,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException {
return getInternal(uriInfo, securityContext, id, fieldsParam, include);
}
@PATCH
@Path("/{id}")
@Operation(
operationId = "patchWebAnalyticEventTypeById",
summary = "Update a web analytic event type",
tags = "WebAnalyticEvent",
description = "Update a web analytic event type.",
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
public Response updateDescription(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") UUID id,
@RequestBody(
description = "JsonPatch with array of operations",
content =
@Content(
mediaType = MediaType.APPLICATION_JSON_PATCH_JSON,
examples = {
@ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]")
}))
JsonPatch patch)
throws IOException {
return patchInternal(uriInfo, securityContext, id, patch);
}
@DELETE
@Path("/{id}")
@Operation(
operationId = "deleteWebAnalyticEventTypeById",
summary = "delete a web analytic event type",
tags = "WebAnalyticEvent",
description = "Delete a web analytic event type by id.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "Web Analytic event for instance {id} is not found")
})
public Response delete(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Hard delete the entity. (Default = `false`)")
@QueryParam("hardDelete")
@DefaultValue("false")
boolean hardDelete,
@Parameter(description = "Web Analytic event Id", schema = @Schema(type = "UUID")) @PathParam("id") UUID id)
throws IOException {
return delete(uriInfo, securityContext, id, false, hardDelete, true);
}
@GET
@Path("/name/{name}")
@Operation(
operationId = "getWebAnalyticEventTypeByName",
summary = "Get a web analytic event type by Name",
tags = "WebAnalyticEvent",
description = "Get a web analytic event type by Name.",
responses = {
@ApiResponse(
responseCode = "200",
description = "A web analytic event type",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = WebAnalyticEvent.class))),
@ApiResponse(responseCode = "404", description = "Web Analytic event type for instance {id} is not found")
})
public WebAnalyticEvent getByName(
@Context UriInfo uriInfo,
@PathParam("name") String name,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException {
return getByNameInternal(uriInfo, securityContext, name, fieldsParam, include);
}
@GET
@Path("/{id}/versions")
@Operation(
operationId = "listAllWebAnalyticEventTypeVersion",
summary = "List web analytic event type versions",
tags = "WebAnalyticEvent",
description = "Get a list of all the version of a web analytic event type by `id`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "List all web analytic event type versions",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class)))
})
public EntityHistory listVersions(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Web Analytic event type Id", schema = @Schema(type = "string")) @PathParam("id")
UUID id)
throws IOException {
return super.listVersionsInternal(securityContext, id);
}
@GET
@Path("/{id}/versions/{version}")
@Operation(
operationId = "getSpecificWebAnalyticEventTypeVersion",
summary = "Get a version of the report definition",
tags = "WebAnalyticEvent",
description = "Get a version of the web analytic event type by `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "WebAnalyticEvent",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = WebAnalyticEvent.class))),
@ApiResponse(
responseCode = "404",
description = "Web Analytic event type for instance {id} and version {version} is " + "not found")
})
public WebAnalyticEvent getVersion(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Web Analytic Event type Id", schema = @Schema(type = "string")) @PathParam("id")
UUID id,
@Parameter(
description = "Web Analytic Event type version number in the form `major`.`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version")
String version)
throws IOException {
return super.getVersionInternal(securityContext, id, version);
}
@PUT
@Path("/collect")
@Operation(
operationId = "addWebAnalyticEventData",
summary = "Add web analytic event data",
tags = "WebAnalyticEvent",
description = "Add web analytic event data",
responses = {
@ApiResponse(
responseCode = "200",
description = "Successfully added web analytic event data",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = WebAnalyticEventData.class)))
})
public Response addReportResult(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Valid WebAnalyticEventData webAnalyticEventData)
throws IOException {
authorizer.authorizeAdmin(securityContext, true);
return dao.addWebAnalyticEventData(uriInfo, webAnalyticEventData);
}
@GET
@Path("/collect")
@Operation(
operationId = "getWebAnalyticEventData",
summary = "Retrieve web analytic data",
tags = "WebAnalyticEvent",
description = "Retrieve web analytic data.",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of web analytic data",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = WebAnalyticEventResource.WebAnalyticEventDataList.class)))
})
public ResultList<WebAnalyticEventData> listWebAnalyticEventData(
@Context SecurityContext securityContext,
@Parameter(
description = "Filter web analytic events for a particular event type",
schema = @Schema(type = "string"))
@NonNull
@QueryParam("eventType")
String eventType,
@Parameter(
description = "Filter web analytic events after the given start timestamp",
schema = @Schema(type = "number"))
@NonNull
@QueryParam("startTs")
Long startTs,
@Parameter(
description = "Filter web analytic events before the given end timestamp",
schema = @Schema(type = "number"))
@NonNull
@QueryParam("endTs")
Long endTs)
throws IOException {
return dao.getWebAnalyticEventData(eventType, startTs, endTs);
}
private WebAnalyticEvent getWebAnalyticEvent(CreateWebAnalyticEvent create, String user) throws IOException {
return copy(new WebAnalyticEvent(), create, user)
.withName(create.getName())
.withDisplayName(create.getDisplayName())
.withDescription(create.getDescription())
.withEventType(create.getEventType());
}
}

View File

@ -438,5 +438,16 @@
"Delete", "Delete",
"ViewAll" "ViewAll"
] ]
},
{
"name" : "webAnalyticEvent",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditDisplayName"
]
} }
] ]

View File

@ -0,0 +1,5 @@
{
"name": "PageView",
"description": "Captures web analytic page view events",
"eventType": "PageView"
}

View File

@ -0,0 +1,145 @@
package org.openmetadata.service.resources.analytics;
import static javax.ws.rs.core.Response.Status.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.assertResponseContains;
import java.io.IOException;
import java.text.ParseException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.ws.rs.client.WebTarget;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.openmetadata.schema.analytics.PageViewData;
import org.openmetadata.schema.analytics.WebAnalyticEvent;
import org.openmetadata.schema.analytics.WebAnalyticEventData;
import org.openmetadata.schema.analytics.type.WebAnalyticEventType;
import org.openmetadata.schema.api.tests.CreateWebAnalyticEvent;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationTest;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.util.ResultList;
import org.openmetadata.service.util.TestUtils;
public class WebAnalyticEventResourceTest extends EntityResourceTest<WebAnalyticEvent, CreateWebAnalyticEvent> {
public WebAnalyticEventResourceTest() {
super(
Entity.WEB_ANALYTIC_EVENT,
WebAnalyticEvent.class,
WebAnalyticEventResource.WebAnalyticEventList.class,
"analytics/webAnalyticEvent",
WebAnalyticEventResource.FIELDS);
supportsEmptyDescription = false;
supportsFollowers = false;
supportsAuthorizedMetadataOperations = false;
supportsOwner = false;
}
@Test
void post_web_analytic_event_200(TestInfo test) throws IOException {
CreateWebAnalyticEvent create = createRequest(test);
create.withName("bar").withEventType(WebAnalyticEventType.PAGE_VIEW);
WebAnalyticEvent webAnalyticEvent = createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
webAnalyticEvent = getEntity(webAnalyticEvent.getId(), ADMIN_AUTH_HEADERS);
validateCreatedEntity(webAnalyticEvent, create, ADMIN_AUTH_HEADERS);
}
@Test
void post_web_analytic_event_4x(TestInfo test) throws IOException {
assertResponseContains(
() -> createEntity(createRequest(test).withEventType(null), ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"eventType must not be null");
assertResponseContains(
() -> createEntity(createRequest(test).withName(null), ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"name must not be null");
}
@Test
void put_web_analytic_event_data_200(TestInfo test) throws IOException, ParseException {
WebAnalyticEventData webAnalyticEventData =
new WebAnalyticEventData()
.withTimestamp(TestUtils.dateToTimestamp("2022-10-11"))
.withEventType(WebAnalyticEventType.PAGE_VIEW)
.withEventData(
new PageViewData()
.withHostname("http://localhost")
.withUserId(UUID.randomUUID())
.withSessionId(UUID.randomUUID()));
putWebAnalyticEventData(webAnalyticEventData, ADMIN_AUTH_HEADERS);
ResultList<WebAnalyticEventData> webAnalyticEventDataResultList =
getWebAnalyticEventData(
WebAnalyticEventType.PAGE_VIEW.value(),
TestUtils.dateToTimestamp("2022-10-10"),
TestUtils.dateToTimestamp("2022-10-12"),
ADMIN_AUTH_HEADERS);
verifyWebAnalyticEventData(webAnalyticEventDataResultList, List.of(webAnalyticEventData), 1);
}
@Override
public CreateWebAnalyticEvent createRequest(String name) {
return new CreateWebAnalyticEvent()
.withName(name)
.withDescription(name)
.withEventType(WebAnalyticEventType.PAGE_VIEW);
}
@Override
public void validateCreatedEntity(
WebAnalyticEvent createdEntity, CreateWebAnalyticEvent request, Map<String, String> authHeaders)
throws HttpResponseException {
assertEquals(request.getName(), createdEntity.getName());
assertEquals(request.getDescription(), createdEntity.getDescription());
}
@Override
public void compareEntities(WebAnalyticEvent expected, WebAnalyticEvent updated, Map<String, String> authHeaders)
throws HttpResponseException {
assertEquals(expected.getName(), updated.getName());
assertEquals(expected.getFullyQualifiedName(), updated.getFullyQualifiedName());
assertEquals(expected.getDescription(), updated.getDescription());
}
@Override
public WebAnalyticEvent validateGetWithDifferentFields(WebAnalyticEvent entity, boolean byName)
throws HttpResponseException {
return null;
}
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
return;
}
public static void putWebAnalyticEventData(WebAnalyticEventData data, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = OpenMetadataApplicationTest.getResource("analytics/webAnalyticEvent/collect");
TestUtils.put(target, data, OK, authHeaders);
}
public static ResultList<WebAnalyticEventData> getWebAnalyticEventData(
String eventType, Long start, Long end, Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = OpenMetadataApplicationTest.getResource("analytics/webAnalyticEvent/collect");
target = target.queryParam("startTs", start);
target = target.queryParam("endTs", end);
target = target.queryParam("eventType", eventType);
return TestUtils.get(target, WebAnalyticEventResource.WebAnalyticEventDataList.class, authHeaders);
}
private void verifyWebAnalyticEventData(
ResultList<WebAnalyticEventData> actualWebAnalyticEventData,
List<WebAnalyticEventData> expectedWebAnalyticEventData,
int expectedCount) {
assertEquals(expectedCount, actualWebAnalyticEventData.getPaging().getTotal());
assertEquals(expectedWebAnalyticEventData.size(), actualWebAnalyticEventData.getData().size());
}
}

View File

@ -0,0 +1,16 @@
{
"$id": "https://open-metadata.org/schema/analytics/basic.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Basic",
"description": "This schema defines basic types that are used by analytics classes",
"definitions": {
"webAnalyticEventType": {
"javaType": "org.openmetadata.schema.analytics.type.WebAnalyticEventType",
"description": "event type",
"type": "string",
"enum": [
"PageView"
]
}
}
}

View File

@ -0,0 +1,46 @@
{
"$id": "https://open-metadata.org/schema/analytics/pageViewData.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "pageViewData",
"type": "object",
"javaType": "org.openmetadata.schema.analytics.PageViewData",
"description": "Page view data event",
"properties": {
"fullUrl": {
"description": "complete URL of the page",
"type": "string"
},
"url": {
"description": "url part after the domain specification",
"type": "string"
},
"hostname": {
"description": "domain name",
"type": "string"
},
"language": {
"description": "language set on the page",
"type": "string"
},
"screenSize": {
"description": "Size of the screen",
"type": "string"
},
"userId": {
"description": "OpenMetadata logged in user Id",
"$ref": "../type/basic.json#/definitions/uuid"
},
"sessionId": {
"description": "Unique ID identifying a session",
"$ref": "../type/basic.json#/definitions/uuid"
},
"pageLoadTime": {
"description": "time for the page to load in seconds",
"type": "number"
},
"referrer": {
"description": "referrer URL",
"type": "string"
}
}
}

View File

@ -0,0 +1,72 @@
{
"$id": "https://open-metadata.org/schema/analytics/webAnalyticEvent.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "WebAnalyticEvent",
"description": "Web Analytic Event",
"type": "object",
"javaType": "org.openmetadata.schema.analytics.WebAnalyticEvent",
"javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
"properties": {
"id": {
"description": "Unique identifier of the report.",
"$ref": "../type/basic.json#/definitions/uuid"
},
"name": {
"description": "Name that identifies this event.",
"$ref": "../type/basic.json#/definitions/entityName"
},
"fullyQualifiedName": {
"description": "FullyQualifiedName same as `name`.",
"$ref": "../type/basic.json#/definitions/fullyQualifiedEntityName"
},
"displayName": {
"description": "Display Name that identifies this web analytics event.",
"type": "string"
},
"description": {
"description": "Description of the event.",
"$ref": "../type/basic.json#/definitions/markdown"
},
"eventType": {
"description": "event type",
"$ref": "./basic.json#/definitions/webAnalyticEventType"
},
"version": {
"description": "Metadata version of the entity.",
"$ref": "../type/entityHistory.json#/definitions/entityVersion"
},
"owner": {
"description": "Owner of this report.",
"$ref": "../type/entityReference.json",
"default": null
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../type/basic.json#/definitions/timestamp"
},
"updatedBy": {
"description": "User who performed the update.",
"type": "string"
},
"href": {
"description": "Link to the resource corresponding to this entity.",
"$ref": "../type/basic.json#/definitions/href"
},
"changeDescription": {
"description": "Change that lead to this version of the entity.",
"$ref": "../type/entityHistory.json#/definitions/changeDescription"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
},
"enabled": {
"description": "Weather the event is enable (i.e. data is being collected)",
"type": "boolean",
"default": true
}
},
"required": ["eventType", "name"],
"additionalProperties": false
}

View File

@ -0,0 +1,30 @@
{
"$id": "https://open-metadata.org/schema/analytics/webAnalyticEventData.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "webAnalyticEventData",
"description": "web analytics event data",
"type": "object",
"javaType": "org.openmetadata.schema.analytics.WebAnalyticEventData",
"properties": {
"eventId": {
"description": "Unique identifier of the report.",
"$ref": "../type/basic.json#/definitions/uuid"
},
"timestamp": {
"description": "event timestamp",
"$ref": "../type/basic.json#/definitions/timestamp"
},
"eventType": {
"description": "event type",
"$ref": "./basic.json#/definitions/webAnalyticEventType"
},
"eventData": {
"description": "Web analytic data captured",
"oneOf": [
{"$ref": "./pageViewEvent.json"}
]
}
},
"required": ["eventType"],
"additionalProperties": false
}

View File

@ -0,0 +1,33 @@
{
"$id": "https://open-metadata.org/schema/api/tests/createWebAnalyticEvent.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "CreateWebAnalyticEvent",
"description": "Payload to create a web analytic event",
"type": "object",
"javaType": "org.openmetadata.schema.api.tests.CreateWebAnalyticEvent",
"javaInterfaces": ["org.openmetadata.schema.CreateEntity"],
"properties": {
"name": {
"description": "Name that identifies this report definition.",
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name the report definition.",
"type": "string"
},
"description": {
"description": "Description of the report definition.",
"$ref": "../../type/basic.json#/definitions/markdown"
},
"eventType": {
"description": "dimension(s) and metric(s) for a report",
"$ref": "../../analytics/basic.json#/definitions/webAnalyticEventType"
},
"owner": {
"description": "Owner of this report definition",
"$ref": "../../type/entityReference.json"
}
},
"required": ["name", "eventType"],
"additionalProperties": false
}